Browse Source

Implement an image downloader in Go

The downloads allows to list available images and to download, with a
progress bar 🎉 !
master
Andreas Linz 2 years ago
parent
commit
df0a10220d
Signed by: alinz GPG Key ID: 9BF39809C9DA580C
4 changed files with 164 additions and 2 deletions
  1. +3
    -2
      .gitignore
  2. +17
    -0
      go.mod
  3. +32
    -0
      go.sum
  4. +112
    -0
      raspbian-image.go

+ 3
- 2
.gitignore View File

@ -1,3 +1,4 @@
/wpa_supplicant.conf
/*raspbian*
/*raspbian*.zip*
/*raspbian*.info
/*raspbian*.img

+ 17
- 0
go.mod View File

@ -0,0 +1,17 @@
module github.com/klingtnet/raspi-toolbox
require (
github.com/PuerkitoBio/goquery v1.4.1 // indirect
github.com/andybalholm/cascadia v1.0.0 // indirect
github.com/antchfx/htmlquery v0.0.0-20180925020018-98389addba3d // indirect
github.com/antchfx/xmlquery v0.0.0-20180925013719-07935b1c0f2e // indirect
github.com/antchfx/xpath v0.0.0-20180922041825-3de91f3991a1 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gocolly/colly v1.1.0 // indirect
github.com/kennygrant/sanitize v1.2.4 // indirect
github.com/machinebox/progress v0.2.0 // indirect
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
github.com/temoto/robotstxt v0.0.0-20180810133444-97ee4a9ee6ea // indirect
golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3 // indirect
google.golang.org/appengine v1.2.0 // indirect
)

+ 32
- 0
go.sum View File

@ -0,0 +1,32 @@
github.com/PuerkitoBio/goquery v1.4.1 h1:smcIRGdYm/w7JSbcdeLHEMzxmsBQvl8lhf0dSw2nzMI=
github.com/PuerkitoBio/goquery v1.4.1/go.mod h1:T9ezsOHcCrDCgA8aF1Cqr3sSYbO/xgdy8/R/XiIMAhA=
github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/antchfx/htmlquery v0.0.0-20180925020018-98389addba3d h1:BzVCjAnYYfpdD01lzVKlZ8mCJu+l4IdGGHjye+R8qgQ=
github.com/antchfx/htmlquery v0.0.0-20180925020018-98389addba3d/go.mod h1:MS9yksVSQXls00iXkiMqXr0J+umL/AmxXKuP28SUJM8=
github.com/antchfx/xmlquery v0.0.0-20180925013719-07935b1c0f2e h1:0KotO9Ay0it1o6O+PzfHSke5+q1EWRlUo9MoPO1+J3I=
github.com/antchfx/xmlquery v0.0.0-20180925013719-07935b1c0f2e/go.mod h1:/+CnyD/DzHRnv2eRxrVbieRU/FIF6N0C+7oTtyUtCKk=
github.com/antchfx/xpath v0.0.0-20180922041825-3de91f3991a1 h1:nt4RMjvM9pJ5HR7WkCeAnDz1pgHbLT2vxICalUTWcJU=
github.com/antchfx/xpath v0.0.0-20180922041825-3de91f3991a1/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
github.com/gocolly/colly v1.1.0 h1:B1M8NzjFpuhagut8f2ILUDlWMag+nTx+PWEmPy7RhrE=
github.com/gocolly/colly v1.1.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
github.com/machinebox/progress v0.2.0 h1:7z8+w32Gy1v8S6VvDoOPPBah3nLqdKjr3GUly18P8Qo=
github.com/machinebox/progress v0.2.0/go.mod h1:hl4FywxSjfmkmCrersGhmJH7KwuKl+Ueq9BXkOny+iE=
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
github.com/temoto/robotstxt v0.0.0-20180810133444-97ee4a9ee6ea h1:hH8P1IiDpzRU6ZDbDh/RDnVuezi2oOXJpApa06M0zyI=
github.com/temoto/robotstxt v0.0.0-20180810133444-97ee4a9ee6ea/go.mod h1:aOux3gHPCftJ3KHq6Pz/AlDjYJ7Y+yKfm1gU/3B0u04=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3 h1:dgd4x4kJt7G4k4m93AYLzM8Ni6h2qLTfh9n9vXJT3/0=
golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
google.golang.org/appengine v1.2.0 h1:S0iUepdCWODXRvtE+gcRDd15L+k+k1AiHlMiMjefH24=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

+ 112
- 0
raspbian-image.go View File

@ -0,0 +1,112 @@
package main
import (
"context"
"fmt"
"io"
"log"
"net/http"
"os"
"regexp"
"sort"
"strconv"
"strings"
"time"
"github.com/gocolly/colly"
"github.com/machinebox/progress"
)
func list() {
c := colly.NewCollector(
colly.AllowedDomains("downloads.raspberrypi.org"),
colly.UserAgent("github.com/klingtnet/raspi-toolbox"),
colly.Async(true),
)
c.Limit(&colly.LimitRule{Parallelism: 4})
urls := []string{}
imgMatcher := regexp.MustCompile(".*raspbian-stretch-lite.*\\.zip$")
c.OnHTML("a[href]", func(e *colly.HTMLElement) {
link := e.Attr("href")
if strings.HasPrefix(link, "raspbian_lite") {
c.Visit(e.Request.AbsoluteURL(link))
return
}
if imgMatcher.Match([]byte(link)) {
urls = append(urls, e.Request.AbsoluteURL(link))
}
})
c.Visit("https://downloads.raspberrypi.org/raspbian_lite/images/")
c.Wait()
sort.Strings(urls)
for _, u := range urls {
fmt.Println(u)
}
}
func download(u string) error {
f, err := os.Create("raspbian.zip")
if err != nil {
return err
}
defer f.Close()
resp, err := http.Get(u)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("expected HTTP 200 OK but was %d", resp.StatusCode)
}
size, err := strconv.Atoi(resp.Header.Get("Content-Length"))
if err != nil {
return err
}
pr := progress.NewReader(resp.Body)
go func() {
progressChan := progress.NewTicker(context.Background(), pr, int64(size), 3*time.Second)
for p := range progressChan {
fmt.Printf("\r%.2f%%", p.Percent())
}
}()
_, err = io.Copy(f, pr)
return err
}
func usage() {
fmt.Printf(`%s [list|usage|help|<url>]
usage|help:
Prints this message
ist:
Prints all available images
<url>:
Downloads the image under <url>
`, os.Args[0])
}
func main() {
if len(os.Args) <= 1 {
usage()
os.Exit(1)
}
arg := os.Args[1]
switch arg {
case "usage":
fallthrough
case "help":
usage()
return
case "list":
list()
default:
if err := download(os.Args[1]); err != nil {
log.Fatalln(err)
}
}
}

Loading…
Cancel
Save