From 3c6e3df23c6ffea56fe4138f9e1476fc09cbc2ae Mon Sep 17 00:00:00 2001 From: Martin Thielecke Date: Sat, 9 May 2020 01:51:02 +0200 Subject: [PATCH] initial version of collector --- .gitignore | 2 + channel.go | 76 +++++++++++++++++++++++ follower.go | 96 +++++++++++++++++++++++++++++ main.go | 110 +++++++++++++++++++++++++++++++++ reflect.go | 65 ++++++++++++++++++++ save.go | 42 +++++++++++++ settings.go | 60 ++++++++++++++++++ settings.yml.dist | 3 + stream.go | 87 ++++++++++++++++++++++++++ subs.go | 106 ++++++++++++++++++++++++++++++++ user.go | 151 ++++++++++++++++++++++++++++++++++++++++++++++ 11 files changed, 798 insertions(+) create mode 100644 .gitignore create mode 100644 channel.go create mode 100644 follower.go create mode 100644 main.go create mode 100644 reflect.go create mode 100644 save.go create mode 100644 settings.go create mode 100644 settings.yml.dist create mode 100644 stream.go create mode 100644 subs.go create mode 100644 user.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0e366e2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +settings.yml +.user.json diff --git a/channel.go b/channel.go new file mode 100644 index 0000000..45535a9 --- /dev/null +++ b/channel.go @@ -0,0 +1,76 @@ +package main + +import ( + "encoding/json" + "net/http" + + log "github.com/sirupsen/logrus" +) + +type TwitchChannel struct { + ID string `json:"_id"` + + BroadcasterLanguage string `json:"broadcaster_language"` + BroadcasterType string `json:"broadcaster_type"` + CreatedAt string `json:"created_at"` + DisplayName string `json:"display_name"` + Email string `json:"email"` + Followers int64 `json:"followers"` + Game string `json:"game"` + Language string `json:"language"` + Logo string `json:"logo"` + Mature bool `json:"mature"` + Name string `json:"name"` + Partner bool `json:"partner"` + ProfileBanner string `json:"profile_banner"` + ProfileBannerBackgroundColor string `json:"profile_banner_background_color"` + Status string `json:"status"` + StreamKey string `json:"stream_key"` + UpdatedAt string `json:"updated_at"` + URL string `json:"url"` + VideoBanner string `json:"video_banner"` + Views int64 `json:"views"` +} + +func getChannel(u *User) error { + client := &http.Client{} + req, err := http.NewRequest("GET", "https://api.twitch.tv/kraken/channel", nil) + if err != nil { + log.WithError(err).Error("Unable to create http request to get twitch channel data") + return err + } + + req.Header.Set("Client-ID", settings.ClientID) + req.Header.Set("Accept", "application/vnd.twitchtv.v5+json") + req.Header.Set("Authorization", "OAuth "+u.Token.AccessToken) + + resp, err := client.Do(req) + if err != nil { + log.WithError(err).Error("Unable to get twitch channel data") + return err + } + + defer resp.Body.Close() + c := &TwitchChannel{} + + if err := json.NewDecoder(resp.Body).Decode(c); err != nil { + log.WithError(err).Error("Unable to parse twitch channel data") + return err + } + + u.TwitchChannel = c + + return nil +} + +func (c *TwitchChannel) SaveFiles() { + data, err := fieldsToMap(c) + if err != nil { + log.WithError(err).Error("Unable to convert channel to map") + return + } + + for k, v := range data { + saveContent("channel", k, v) + } +} diff --git a/follower.go b/follower.go new file mode 100644 index 0000000..2b506dc --- /dev/null +++ b/follower.go @@ -0,0 +1,96 @@ +package main + +import ( + "context" + "encoding/json" + "net/http" + "sort" + "strconv" + "strings" + "time" + + log "github.com/sirupsen/logrus" +) + +type TwitchFollowers struct { + Total int64 `json:"total"` + Data []*TwitchFollower `json:"data"` + Pagination *TwitchPagination `json:"pagination"` +} + +type TwitchPagination struct { + Cursor string `json:"cursor"` +} + +type TwitchFollower struct { + FromID string `json:"from_id"` + FromName string `json:"from_name"` + ToID string `json:"to_id"` + ToName string `json:"to_name"` + FollowedAt time.Time `json:"followed_at"` +} + +func getFollows(u *User) { + result := &TwitchFollowers{} + + after := "" + for { + client := twitchOauthConfig.Client(context.Background(), u.Token) + req, err := http.NewRequest("GET", "https://api.twitch.tv/helix/users/follows?to_id="+u.ID+after, nil) + if err != nil { + log.WithError(err).Error("Unable to create http request to get twitch follower data") + return + } + req.Header.Set("Client-ID", settings.ClientID) + + resp, err := client.Do(req) + if err != nil { + log.WithError(err).Error("Unable to get twitch follower data") + return + } + + t := &TwitchFollowers{} + if err := json.NewDecoder(resp.Body).Decode(&t); err != nil { + log.WithError(err).Error("Unable to parse twitch followers data") + } + resp.Body.Close() + + if len(t.Data) == 0 { + break + } + + result.Total = t.Total + result.Data = append(result.Data, t.Data...) + if t.Pagination == nil || t.Pagination.Cursor == "" { + break + } + after = "&after=" + t.Pagination.Cursor + } + + if len(result.Data) < 1 { + log.Info("No followers") + return + } + + u.TwitchFollowers = result +} + +func (f *TwitchFollowers) SaveFiles() { + saveContent("followers", "total", strconv.FormatInt(f.Total, 10)) + saveJSON("followers", "complete_list", f) + sort.Slice(f.Data, func(i, j int) bool { + return f.Data[i].FollowedAt.Before(f.Data[j].FollowedAt) + }) + start := len(f.Data) - 10 + if start < 0 { + start = 0 + } + lastFollowers := f.Data[start:] + + var lastFollowerSlice []string + for _, v := range lastFollowers { + lastFollowerSlice = append(lastFollowerSlice, v.FromName) + } + + saveContent("followers", "last_ten", strings.Join(lastFollowerSlice, "\n")) +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..7c2f504 --- /dev/null +++ b/main.go @@ -0,0 +1,110 @@ +package main + +import ( + "context" + "net/http" + "time" + + "github.com/Luzifer/rconfig" + gorilla "github.com/gorilla/mux" + log "github.com/sirupsen/logrus" + "golang.org/x/oauth2" +) + +var cfg = struct { + SettingsFile string `default:"settings.yml" flag:"settings-file" description:"Path to settings file"` +}{} + +func main() { + rconfig.Parse(&cfg) + settingsUpdater() + + loadUser() + + log.Infof("User: %+v", user) + + mux := gorilla.NewRouter() + mux.HandleFunc("/login", handleTwitchLogin) + mux.HandleFunc("/callback", handleTwitchCallback) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("Hi!")) + }) + + go func() { + handleSaves() + c := time.Tick(20 * time.Second) + for range c { + handleSaves() + } + }() + + log.Info("Starting webserver...") + log.Fatal(http.ListenAndServe(":8083", mux)) +} + +func handleSaves() { + if user == nil { + return + } + + getUser(user) + + if user.TwitchUser != nil { + user.TwitchUser.SaveFiles() + } + + getChannel(user) + if user.TwitchChannel != nil { + user.TwitchChannel.SaveFiles() + } + + getFollows(user) + if user.TwitchFollowers != nil { + user.TwitchFollowers.SaveFiles() + } + + getSubs(user) + if user.TwitchSubscriptions != nil { + user.TwitchSubscriptions.SaveFiles() + } + + getStreams(user) + if user.TwitchStream != nil { + user.TwitchStream.SaveFiles() + } +} + +func handleTwitchLogin(w http.ResponseWriter, r *http.Request) { + url := twitchOauthConfig.AuthCodeURL(settings.VerificationToken, oauth2.AccessTypeOffline) + http.Redirect(w, r, url, http.StatusTemporaryRedirect) +} + +func handleTwitchCallback(w http.ResponseWriter, r *http.Request) { + twitchAuthToToken(r.FormValue("state"), r.FormValue("code"), w, r) + http.Redirect(w, r, "/", http.StatusTemporaryRedirect) +} + +func twitchAuthToToken(state string, code string, w http.ResponseWriter, r *http.Request) { + if state != settings.VerificationToken { + log.Fatal("invalid oauth state") + } + + timeoutCtx, cancel := context.WithTimeout(oauth2.NoContext, 5*time.Second) + defer cancel() + + token, err := twitchOauthConfig.Exchange(timeoutCtx, code, oauth2.AccessTypeOffline) + if err != nil { + http.Redirect(w, r, "/", http.StatusTemporaryRedirect) + log.WithError(err).Error("code exchange failed") + return + } + u, err := createUser(token) + if err != nil { + log.WithError(err).Error("Unable to get user") + return + } + u.Token = token + user = u + log.Infof("User: %+v, User: %+v", user, user.TwitchUser) + saveUser() +} diff --git a/reflect.go b/reflect.go new file mode 100644 index 0000000..fdbc1af --- /dev/null +++ b/reflect.go @@ -0,0 +1,65 @@ +package main + +import ( + "reflect" + "strconv" + "strings" + "time" + + "github.com/pkg/errors" +) + +func fieldsToMap(in interface{}) (map[string]string, error) { + if reflect.TypeOf(in).Kind() != reflect.Ptr { + return nil, errors.New("Non-pointer given") + } + + if kind := reflect.ValueOf(in).Elem().Kind(); kind != reflect.Struct { + return nil, errors.Errorf("Non-struct given: %s", kind) + } + + var out = map[string]string{} + + st := reflect.ValueOf(in).Elem() + for i := 0; i < st.NumField(); i++ { + valField := st.Field(i) + typeField := st.Type().Field(i) + + jsonTag := strings.Split(typeField.Tag.Get("json"), ",")[0] + if jsonTag == "" { + // Empty tag, skip + continue + } + + switch typeField.Type { + case reflect.TypeOf(time.Time{}): + out[jsonTag] = valField.Addr().Interface().(*time.Time).String() + continue + } + + switch typeField.Type.Kind() { + + case reflect.Bool: + out[jsonTag] = strconv.FormatBool(valField.Bool()) + + case reflect.Int, reflect.Int64: + out[jsonTag] = strconv.FormatInt(valField.Int(), 10) + + case reflect.String: + out[jsonTag] = valField.String() + + case reflect.Slice: + switch typeField.Type.Elem().Kind() { + case reflect.String: + res := valField.Addr().Interface().(*[]string) + out[jsonTag] = strings.Join(*res, "\n") + } + + default: + return nil, errors.Errorf("Unhandled field type: %s", typeField.Type.String()) + + } + } + + return out, nil +} diff --git a/save.go b/save.go new file mode 100644 index 0000000..573a607 --- /dev/null +++ b/save.go @@ -0,0 +1,42 @@ +package main + +import ( + "encoding/json" + "io/ioutil" + "os" + "path" + + log "github.com/sirupsen/logrus" +) + +func saveContent(kind, filename, content string) { + p := path.Join(kind, filename+".txt") + if err := os.MkdirAll(kind, 0777); err != nil { + log.WithField("path", p).WithError(err).Error("Unable to create directory") + return + } + + if err := ioutil.WriteFile(p, []byte(content), 0777); err != nil { + log.WithError(err).Error("Unable to write content") + } +} + +func saveJSON(kind, filename string, data interface{}) { + p := path.Join(kind, filename+".json") + if err := os.MkdirAll(kind, 0777); err != nil { + log.WithField("path", p).WithError(err).Error("Unable to create directory") + return + } + + f, err := os.Create(p) + if err != nil { + log.WithField("path", p).WithError(err).Error("Unable to create file") + return + } + defer f.Close() + + if err := json.NewEncoder(f).Encode(data); err != nil { + log.WithField("path", p).WithError(err).Error("Unable to encode json") + return + } +} diff --git a/settings.go b/settings.go new file mode 100644 index 0000000..9f8011c --- /dev/null +++ b/settings.go @@ -0,0 +1,60 @@ +package main + +import ( + "bytes" + "io/ioutil" + "time" + + log "github.com/sirupsen/logrus" + "golang.org/x/oauth2" + "golang.org/x/oauth2/twitch" + "gopkg.in/yaml.v2" +) + +var ( + settings *Settings + twitchOauthConfig *oauth2.Config +) + +type Settings struct { + ClientID string `yaml:"client_id"` + ClientSecret string `yaml:"client_secret"` + VerificationToken string `yaml:"verification_token"` +} + +func loadSettings() { + data, err := ioutil.ReadFile(cfg.SettingsFile) + if err != nil { + log.WithError(err).Errorf("Unable to read %s", cfg.SettingsFile) + return + } + + s := &Settings{} + b := bytes.NewBuffer(data) + if err := yaml.NewDecoder(b).Decode(s); err != nil { + log.WithError(err).Errorf("Unable to decode %s", cfg.SettingsFile) + return + } + + settings = s + + twitchOauthConfig = &oauth2.Config{ + RedirectURL: "http://localhost:8080/callback", + ClientID: settings.ClientID, + ClientSecret: settings.ClientSecret, + Scopes: []string{"channel:read:subscriptions", "user:read:broadcast", "chat:read", "chat:edit", "channel_read", "channel_editor", "channel_subscriptions", "channel:moderate", "bits:read", "channel:read:redemptions"}, + Endpoint: twitch.Endpoint, + } + +} + +func settingsUpdater() { + loadSettings() + + go func() { + c := time.Tick(time.Minute) + for range c { + loadSettings() + } + }() +} diff --git a/settings.yml.dist b/settings.yml.dist new file mode 100644 index 0000000..40325f6 --- /dev/null +++ b/settings.yml.dist @@ -0,0 +1,3 @@ +client_id: "your twitch client id" +client_secret: "your twitch client secret" +verification_token: "a random string!" diff --git a/stream.go b/stream.go new file mode 100644 index 0000000..e569c15 --- /dev/null +++ b/stream.go @@ -0,0 +1,87 @@ +package main + +import ( + "context" + "encoding/json" + "net/http" + "time" + + log "github.com/sirupsen/logrus" +) + +type TwitchStreams struct { + Data []*TwitchStream `json:"data"` + Pagination *TwitchPagination `json:"pagination"` +} + +type TwitchStream struct { + GameID string `json:"game_id"` + ID string `json:"id"` + Language string `json:"language"` + StartedAt time.Time `json:"started_at"` + TagIDs []string `json:"tag_ids"` + ThumbnailURL string `json:"thumbnail_url"` + Title string `json:"title"` + Type string `json:"type"` + UserID string `json:"user_id"` + UserName string `json:"user_name"` + ViewerCount int `json:"viewer_count"` +} + +func getStreams(u *User) { + result := &TwitchStreams{} + + after := "" + for { + client := twitchOauthConfig.Client(context.Background(), u.Token) + req, err := http.NewRequest("GET", "https://api.twitch.tv/helix/streams?user_id="+u.ID+after, nil) + if err != nil { + log.WithError(err).Error("Unable to create http request to get twitch streams data") + return + } + req.Header.Set("Client-ID", settings.ClientID) + + resp, err := client.Do(req) + if err != nil { + log.WithError(err).Error("Unable to get twitch stream data") + return + } + + t := &TwitchStreams{} + if err := json.NewDecoder(resp.Body).Decode(&t); err != nil { + log.WithError(err).Error("Unable to parse twitch streams data") + } + resp.Body.Close() + + if len(t.Data) == 0 { + break + } + + result.Data = append(result.Data, t.Data...) + if t.Pagination == nil || t.Pagination.Cursor == "" { + break + } + after = "&after=" + t.Pagination.Cursor + } + + if len(result.Data) < 1 { + log.Info("No streams") + return + } + + if len(result.Data) > 0 { + u.TwitchStream = result.Data[0] + } +} + +func (s *TwitchStream) SaveFiles() { + data, err := fieldsToMap(s) + if err != nil { + log.WithError(err).Error("Unable to convert stream to map") + return + } + + for k, v := range data { + saveContent("stream", k, v) + } +} diff --git a/subs.go b/subs.go new file mode 100644 index 0000000..b2e0b8f --- /dev/null +++ b/subs.go @@ -0,0 +1,106 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "sort" + "strconv" + "strings" + "time" + + log "github.com/sirupsen/logrus" +) + +type TwitchSubscriptions struct { + Total int64 `json:"_total"` + Subscriptions []*TwitchSubscription `json:"subscriptions"` +} + +type TwitchSubscription struct { + ID string `json:"_id"` + CreatedAt time.Time `json:"created_at"` + IsGift bool `json:"is_gift"` + SubPlan string `json:"sub_plan"` + SubPlanName string `json:"sub_plan_name"` + User *TwitchSubUser `json:"user"` +} + +type TwitchSubUser struct { + ID string `json:"_id"` + Bio string `json:"bio"` + CreatedAt time.Time `json:"created_at"` + DisplayName string `json:"display_name"` + Logo string `json:"logo"` + Name string `json:"name"` + Type string `json:"type"` + UpdatedAt time.Time `json:"updated_at"` +} + +func getSubs(u *User) { + result := &TwitchSubscriptions{} + + limit := 100 + offset := 0 + + for { + client := http.Client{} + req, err := http.NewRequest("GET", fmt.Sprintf("https://api.twitch.tv/kraken/channels/%s/subscriptions?limit=%d&offset=%d", u.TwitchChannel.ID, limit, offset), nil) + if err != nil { + log.WithError(err).Error("Unable to create http request to get twitch subs data") + return + } + + req.Header.Set("Client-ID", settings.ClientID) + req.Header.Set("Accept", "application/vnd.twitchtv.v5+json") + req.Header.Set("Authorization", "OAuth "+u.Token.AccessToken) + + resp, err := client.Do(req) + if err != nil { + log.WithError(err).Error("Unable to get twitch subs data") + return + } + + t := &TwitchSubscriptions{} + if err := json.NewDecoder(resp.Body).Decode(&t); err != nil { + log.WithError(err).Error("Unable to parse twitch followers data") + } + resp.Body.Close() + + if len(t.Subscriptions) == 0 { + break + } + + result.Total = t.Total + result.Subscriptions = append(result.Subscriptions, t.Subscriptions...) + + offset += limit + } + + if len(result.Subscriptions) < 1 { + log.Info("No Subs") + return + } + + u.TwitchSubscriptions = result +} +func (s *TwitchSubscriptions) SaveFiles() { + saveContent("subscriptions", "total", strconv.FormatInt(s.Total, 10)) + saveJSON("subscriptions", "complete_list", s) + + sort.Slice(s.Subscriptions, func(i, j int) bool { + return s.Subscriptions[i].CreatedAt.Before(s.Subscriptions[j].CreatedAt) + }) + start := len(s.Subscriptions) - 10 + if start < 0 { + start = 0 + } + lastSubs := s.Subscriptions[start:] + + var lastSubsSlice []string + for _, v := range lastSubs { + lastSubsSlice = append(lastSubsSlice, v.User.DisplayName) + } + + saveContent("subscriptions", "last_ten", strings.Join(lastSubsSlice, "\n")) +} diff --git a/user.go b/user.go new file mode 100644 index 0000000..f923910 --- /dev/null +++ b/user.go @@ -0,0 +1,151 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "io/ioutil" + "net/http" + "os" + + "github.com/go-irc/irc" + log "github.com/sirupsen/logrus" + "golang.org/x/oauth2" +) + +var ( + user *User +) + +type User struct { + ID string + Name string + DisplayName string + Token *oauth2.Token + IRCClient *irc.Client `json:"-"` + TwitchUser *TwitchUser `json:"-"` + TwitchChannel *TwitchChannel `json:"-"` + TwitchFollowers *TwitchFollowers `json:"-"` + TwitchSubscriptions *TwitchSubscriptions `json:"-"` + TwitchStream *TwitchStream `json:"-"` +} + +type TwitchUser struct { + ID string `json:"id"` + Login string `json:"login"` + DisplayName string `json:"display_name"` + Type string `json:"type"` + BroadcasterType string `json:"broadcaster_type"` + Description string `json:"description"` + ProfileImage string `json:"profile_image_url"` + OfflineImageURL string `json:"offline_image_url"` + ViewCount int64 `json:"view_count"` +} + +func createUser(token *oauth2.Token) (*User, error) { + u := &User{ + Token: token, + } + + if err := getUser(u); err != nil { + log.WithError(err).Error("Unable to get user") + return nil, err + } + + u.ID = u.TwitchUser.ID + u.Name = u.TwitchUser.Login + u.DisplayName = u.TwitchUser.DisplayName + + return u, nil +} + +func getUser(u *User) error { + client := twitchOauthConfig.Client(context.Background(), u.Token) + req, err := http.NewRequest("GET", "https://api.twitch.tv/helix/users", nil) + if err != nil { + log.WithError(err).Error("Unable to create http request to get twitch user data") + return err + } + + req.Header.Set("Client-ID", settings.ClientID) + resp, err := client.Do(req) + if err != nil { + log.WithError(err).Error("Unable to get twitch user data") + return err + } + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.WithError(err).Error("Unable to get response body for user data") + return err + } + + users := map[string][]*TwitchUser{} + + if err := json.Unmarshal(b, &users); err != nil { + log.WithError(err).Error("Unable to parse twitch user data") + return err + } + + u.TwitchUser = users["data"][0] + + return nil +} + +func saveUser() { + log.Info("Save user") + f, err := os.OpenFile(".user.json", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + log.WithError(err).Fatal("Unable to open file") + } + defer f.Close() + + if err := json.NewEncoder(f).Encode(user); err != nil { + log.WithError(err).Fatal("Unable to encode users data") + } +} + +func loadUser() { + data, err := ioutil.ReadFile(".user.json") + if err != nil { + log.WithError(err).Error("Unable to read .users.json") + return + } + + var u *User + + b := bytes.NewBuffer(data) + if err := json.NewDecoder(b).Decode(&u); err != nil { + log.WithError(err).Error("Unable to decode .users.json") + return + } + + if u == nil { + log.Warning("User not existent") + return + } + + if err := getUser(u); err != nil { + log.WithError(err).Error("Unable to get user information") + return + } + + if err := getChannel(u); err != nil { + log.WithError(err).Error("Unable to get channel information") + return + } + + user = u +} + +func (t *TwitchUser) SaveFiles() { + data, err := fieldsToMap(t) + if err != nil { + log.WithError(err).Error("Unable to convert user to map") + return + } + + for k, v := range data { + saveContent("user", k, v) + } +}