scrape/graph/graph.go

   1 package graph
   2
   3 import (
   4         "git.jacobcasper.com/brackets/env"
   5         "github.com/zmb3/spotify"
   6         "log"
   7         "net/http"
   8         "net/url"
   9         "time"
  10 )
  11
  12 func Scrape(env *env.Env) {
  13 infinite:
  14         for {
  15                 time.Sleep(time.Second * 5)
  16                 rows, err := env.Db.Db.Query(`
  17 SELECT ID
  18 FROM ARTIST
  19 WHERE ID NOT IN (
  20   SELECT ARTIST_ID
  21         FROM SCRAPED_ARTIST
  22         WHERE SCRAPED == 1
  23 )`)
  24                 if err != nil {
  25                         log.Print(err)
  26                         continue infinite
  27                 }
  28                 defer rows.Close()
  29
  30                 var artistId string
  31                 for rows.Next() {
  32                         if err := rows.Scan(&artistId); err != nil {
  33                                 log.Print(err)
  34                                 continue infinite
  35                         }
  36
  37                         artists, err := env.C.GetRelatedArtists(spotify.ID(artistId))
  38                         if err != nil {
  39                                 log.Print(err)
  40                                 continue infinite
  41                         }
  42
  43                         success := true
  44                 postArtists:
  45                         for _, artist := range artists {
  46                                 row := env.Db.Db.QueryRow(`
  47 SELECT EXISTS (
  48         SELECT 1
  49         FROM ARTIST
  50         WHERE ID = ?
  51 )
  52 `,
  53                                         artist.ID)
  54                                 var exists bool
  55                                 if err := row.Scan(&exists); err != nil {
  56                                         // We don't care, this was a short circuit check
  57                                 }
  58                                 if exists {
  59                                         continue postArtists
  60                                 }
  61
  62                                 resp, err := http.PostForm("http://localhost:8080/artist/add", url.Values{"id": {string(artist.ID)}})
  63                                 if err != nil {
  64                                         log.Print(err)
  65                                         success = false
  66                                         continue postArtists
  67                                 }
  68                                 if resp.StatusCode != http.StatusCreated {
  69                                         success = false
  70                                 }
  71                         }
  72
  73                         if success {
  74                                 env.Db.Mu.Lock()
  75                                 env.Db.Db.Exec(`
  76 REPLACE INTO SCRAPED_ARTIST (ARTIST_ID, SCRAPED)
  77 VALUES (?, 1)`,
  78                                         string(artistId))
  79                                 env.Db.Mu.Unlock()
  80                         }
  81                 }
  82         }
  83 }