Commit | Line | Data |
---|---|---|
bd7c6d1e JC |
1 | package graph |
2 | ||
3 | import ( | |
4 | "git.jacobcasper.com/brackets/env" | |
5 | "github.com/zmb3/spotify" | |
6 | "log" | |
7 | "net/http" | |
8 | "net/url" | |
9 | "time" | |
10 | ) | |
11 | ||
12 | func Scrape(env *env.Env) { | |
13 | infinite: | |
14 | for { | |
15 | time.Sleep(time.Second * 5) | |
16 | rows, err := env.Db.Db.Query(` | |
17 | SELECT ID | |
18 | FROM ARTIST | |
19 | WHERE ID NOT IN ( | |
20 | SELECT ARTIST_ID | |
21 | FROM SCRAPED_ARTIST | |
22 | WHERE SCRAPED == 1 | |
9a3c373c JC |
23 | )`, |
24 | ) | |
bd7c6d1e JC |
25 | if err != nil { |
26 | log.Print(err) | |
27 | continue infinite | |
28 | } | |
bd7c6d1e JC |
29 | |
30 | var artistId string | |
31 | for rows.Next() { | |
32 | if err := rows.Scan(&artistId); err != nil { | |
33 | log.Print(err) | |
27d4a8e0 | 34 | rows.Close() |
bd7c6d1e JC |
35 | continue infinite |
36 | } | |
37 | ||
38 | artists, err := env.C.GetRelatedArtists(spotify.ID(artistId)) | |
39 | if err != nil { | |
40 | log.Print(err) | |
27d4a8e0 | 41 | rows.Close() |
bd7c6d1e JC |
42 | continue infinite |
43 | } | |
44 | ||
45 | success := true | |
46 | postArtists: | |
47 | for _, artist := range artists { | |
08c4c2a9 JC |
48 | row := env.Db.Db.QueryRow(` |
49 | SELECT EXISTS ( | |
50 | SELECT 1 | |
51 | FROM ARTIST | |
52 | WHERE ID = ? | |
53 | ) | |
54 | `, | |
9a3c373c JC |
55 | artist.ID, |
56 | ) | |
08c4c2a9 JC |
57 | var exists bool |
58 | if err := row.Scan(&exists); err != nil { | |
59 | // We don't care, this was a short circuit check | |
60 | } | |
61 | if exists { | |
62 | continue postArtists | |
63 | } | |
64 | ||
bd7c6d1e | 65 | resp, err := http.PostForm("http://localhost:8080/artist/add", url.Values{"id": {string(artist.ID)}}) |
76f51fa5 | 66 | resp.Body.Close() |
bd7c6d1e JC |
67 | if err != nil { |
68 | log.Print(err) | |
69 | success = false | |
70 | continue postArtists | |
71 | } | |
72 | if resp.StatusCode != http.StatusCreated { | |
73 | success = false | |
74 | } | |
75 | } | |
76 | ||
77 | if success { | |
78 | env.Db.Mu.Lock() | |
79 | env.Db.Db.Exec(` | |
80 | REPLACE INTO SCRAPED_ARTIST (ARTIST_ID, SCRAPED) | |
81 | VALUES (?, 1)`, | |
9a3c373c JC |
82 | string(artistId), |
83 | ) | |
bd7c6d1e JC |
84 | env.Db.Mu.Unlock() |
85 | } | |
86 | } | |
87 | } | |
88 | } |