mthie
/
twitch_data_collector
Archived
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

fix twitch APIs and usage values

This commit is contained in:
Martin Thielecke 2022-05-07 13:27:37 +02:00
parent 780d258cb4
commit 1bb87788f8
Signed by: mthie
GPG Key ID: D1D25A85C8604DFB
5 changed files with 140 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
channel.go
View File

@ -1,72 +1,73 @@
package main
import (
"context"
"encoding/json"
"io/ioutil"
"net/http"
log "github.com/sirupsen/logrus"
)
type TwitchChannel struct {
ID string `json:"_id"`
BroadcasterLanguage string `json:"broadcaster_language"`
BroadcasterType string `json:"broadcaster_type"`
CreatedAt string `json:"created_at"`
DisplayName string `json:"display_name"`
Email string `json:"email"`
Followers int64 `json:"followers"`
Game string `json:"game"`
Language string `json:"language"`
Logo string `json:"logo"`
Mature bool `json:"mature"`
Name string `json:"name"`
Partner bool `json:"partner"`
ProfileBanner string `json:"profile_banner"`
ProfileBannerBackgroundColor string `json:"profile_banner_background_color"`
Status string `json:"status"`
StreamKey string `json:"stream_key"`
UpdatedAt string `json:"updated_at"`
URL string `json:"url"`
VideoBanner string `json:"video_banner"`
Views int64 `json:"views"`
type TwitchChannels struct {
Data []*TwitchChannel `json:"data"`
Pagination *TwitchPagination `json:"pagination"`
}
func getChannel(u *User) error {
client := &http.Client{}
req, err := http.NewRequest("GET", "https://api.twitch.tv/kraken/channel", nil)
if err != nil {
log.WithError(err).Error("Unable to create http request to get twitch channel data")
return err
type TwitchChannel struct {
BroadcasterID string `json:"broadcaster_id"`
BroadcasterLogin string `json:"broadcaster_login"`
BroadcasterName string `json:"broadcaster_name"`
BroadcasterLanguage string `json:"broadcaster_language"`
GameID string `json:"game_id"`
GameName string `json:"game_name"`
Title string `json:"title"`
Delay int `json:"delay"`
}
func getChannel(u *User) {
result := &TwitchChannels{}
after := ""
for {
client := twitchOauthConfig.Client(context.Background(), u.Token)
req, err := http.NewRequest("GET", "https://api.twitch.tv/helix/channels?broadcaster_id="+u.ID+after, nil)
if err != nil {
log.WithError(err).Error("Unable to create http request to get twitch streams data")
return
}
req.Header.Set("Client-ID", settings.ClientID)
resp, err := client.Do(req)
if err != nil {
log.WithError(err).Error("Unable to get twitch stream data")
return
}
t := &TwitchChannels{}
if err := json.NewDecoder(resp.Body).Decode(&t); err != nil {
log.WithError(err).Error("Unable to parse twitch streams data")
}
resp.Body.Close()
if len(t.Data) == 0 {
break
}
result.Data = append(result.Data, t.Data...)
if t.Pagination == nil || t.Pagination.Cursor == "" {
break
}
after = "&after=" + t.Pagination.Cursor
}
req.Header.Set("Client-ID", settings.ClientID)
req.Header.Set("Accept", "application/vnd.twitchtv.v5+json")
req.Header.Set("Authorization", "OAuth "+u.Token.AccessToken)
resp, err := client.Do(req)
if err != nil {
log.WithError(err).Error("Unable to get twitch channel data")
return err
if len(result.Data) < 1 {
log.Info("No streams")
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.WithError(err).Error("Unable to read response")
if len(result.Data) > 0 {
u.TwitchChannel = result.Data[0]
}
c := &TwitchChannel{}
if err := json.Unmarshal(b, c); err != nil {
log.WithError(err).Error("Unable to parse twitch channel data")
log.Infof("Data: %s", string(b))
return err
}
u.TwitchChannel = c
return nil
}
func (c *TwitchChannel) SaveFiles() {

47
follower.go
View File

@ -1,13 +1,10 @@
package main
import (
"bytes"
"context"
"encoding/json"
"io/ioutil"
"net/http"
"os"
"strings"
"strconv"
"time"
log "github.com/sirupsen/logrus"
@ -81,36 +78,22 @@ func getFollows(u *User, max int) {
u.TwitchFollowers = result
}
func readActiveUsers() map[string]bool {
res, err := ioutil.ReadFile(os.Args[1])
if err != nil {
log.WithError(err).Fatal("Unable to read active users")
}
data := make(map[string]bool)
if err := json.NewDecoder(bytes.NewBuffer(res)).Decode(&data); err != nil {
log.WithError(err).Fatal("Unable to decode json")
}
return data
}
func (f *TwitchFollowers) SaveFiles() {
fs := []string{}
active := readActiveUsers()
ids := []string{}
saveContent("followers", "total", strconv.FormatInt(f.Total, 10))
saveJSON("followers", "complete_list", f)
for _, follower := range f.Data {
name := strings.ToLower(follower.FromName)
id := follower.FromID
if _, ok := active[name]; ok {
continue
/*
fs := []string{}
ids := []string{}
for _, follower := range f.Data {
name := strings.ToLower(follower.FromName)
id := follower.FromID
fs = append(fs, name)
ids = append(ids, id)
}
fs = append(fs, name)
ids = append(ids, id)
}
saveJSON("followers", "inactive", fs)
saveContent("followers", "inactive_ids", strings.Join(ids, "\n"))
saveJSON("followers", "inactive", fs)
saveContent("followers", "inactive_ids", strings.Join(ids, "\n"))
*/
}

37
main.go
View File

@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"net/http"
"os"
"time"
"github.com/Luzifer/rconfig"
@ -49,7 +48,18 @@ func main() {
go func() {
if user != nil {
handleSaves()
os.Exit(0)
}
interval := 20 * time.Second
if settings.UpdateInterval > 10*time.Second {
interval = settings.UpdateInterval
}
log.Infof("Update interval: %s", interval)
c := time.Tick(interval)
for range c {
if user == nil {
continue
}
handleSaves()
}
}()
@ -63,16 +73,39 @@ func handleSaves() {
}
getUser(user)
if user.TwitchUser != nil {
user.TwitchUser.SaveFiles()
}
getChannel(user)
if user.TwitchChannel != nil {
user.TwitchChannel.SaveFiles()
}
maxFollower := -1
if settings.MaxFollowers > 0 {
maxFollower = settings.MaxFollowers
}
log.Infof("Fetch max follower: %d", maxFollower)
getFollows(user, maxFollower)
if user.TwitchFollowers != nil {
user.TwitchFollowers.SaveFiles()
}
maxSubs := -1
if settings.MaxSubs > 0 {
maxSubs = settings.MaxSubs
}
log.Infof("Fetch max subs: %d", maxSubs)
getSubs(user, maxSubs)
if user.TwitchSubscriptions != nil {
user.TwitchSubscriptions.SaveFiles()
}
getStreams(user)
if user.TwitchStream != nil {
user.TwitchStream.SaveFiles()
}
}
func handleTwitchLogin(w http.ResponseWriter, r *http.Request) {

92
subs.go
View File

@ -1,109 +1,83 @@
package main
import (
"context"
"encoding/json"
"fmt"
"net/http"
"sort"
"strconv"
"strings"
"time"
log "github.com/sirupsen/logrus"
)
type TwitchSubscriptions struct {
Total int64 `json:"_total"`
Subscriptions []*TwitchSubscription `json:"subscriptions"`
Data []*TwitchSubscription `json:"data"`
Pagination *TwitchPagination `json:"pagination"`
Total int64 `json:"total"`
Points int64 `json:"points"`
}
type TwitchSubscription struct {
ID string `json:"_id"`
CreatedAt time.Time `json:"created_at"`
IsGift bool `json:"is_gift"`
SubPlan string `json:"sub_plan"`
SubPlanName string `json:"sub_plan_name"`
User *TwitchSubUser `json:"user"`
}
type TwitchSubUser struct {
ID string `json:"_id"`
Bio string `json:"bio"`
CreatedAt time.Time `json:"created_at"`
DisplayName string `json:"display_name"`
Logo string `json:"logo"`
Name string `json:"name"`
Type string `json:"type"`
UpdatedAt time.Time `json:"updated_at"`
BroadcasterID string `json:"broadcaster_id"`
BroadcasterLogin string `json:"broadcaster_login"`
BroadcasterName string `json:"broadcaster_name"`
GifterID string `json:"gifter_id"`
GifterLogin string `json:"gifter_login"`
GifterName string `json:"gifter_name"`
IsGift bool `json:"is_gift"`
Tier string `json:"gift"`
PlanName string `json:"plan_name"`
UserID string `json:"user_id"`
UserLogin string `json:"user_login"`
UserName string `json:"user_name"`
}
func getSubs(u *User, max int) {
result := &TwitchSubscriptions{}
limit := 100
offset := 0
after := ""
for {
client := http.Client{}
req, err := http.NewRequest("GET", fmt.Sprintf("https://api.twitch.tv/kraken/channels/%s/subscriptions?limit=%d&offset=%d", u.TwitchChannel.ID, limit, offset), nil)
client := twitchOauthConfig.Client(context.Background(), u.Token)
req, err := http.NewRequest("GET", "https://api.twitch.tv/helix/subscriptions?broadcaster_id="+u.ID+after, nil)
if err != nil {
log.WithError(err).Error("Unable to create http request to get twitch subs data")
log.WithError(err).Error("Unable to create http request to get twitch streams data")
return
}
req.Header.Set("Client-ID", settings.ClientID)
req.Header.Set("Accept", "application/vnd.twitchtv.v5+json")
req.Header.Set("Authorization", "OAuth "+u.Token.AccessToken)
resp, err := client.Do(req)
if err != nil {
log.WithError(err).Error("Unable to get twitch subs data")
log.WithError(err).Error("Unable to get twitch stream data")
return
}
t := &TwitchSubscriptions{}
if err := json.NewDecoder(resp.Body).Decode(&t); err != nil {
log.WithError(err).Error("Unable to parse twitch followers data")
log.WithError(err).Error("Unable to parse twitch streams data")
}
resp.Body.Close()
if len(t.Subscriptions) == 0 {
if len(t.Data) == 0 {
break
}
result.Total = t.Total
result.Subscriptions = append(result.Subscriptions, t.Subscriptions...)
if max > -1 && len(result.Subscriptions) >= max {
result.Data = append(result.Data, t.Data...)
if t.Pagination == nil || t.Pagination.Cursor == "" {
break
}
offset += limit
after = "&after=" + t.Pagination.Cursor
}
if len(result.Subscriptions) < 1 {
log.Info("No Subs")
if len(result.Data) < 1 {
log.Info("No streams")
return
}
u.TwitchSubscriptions = result
if len(result.Data) > 0 {
u.TwitchSubscriptions = result
}
}
func (s *TwitchSubscriptions) SaveFiles() {
saveContent("subscriptions", "total", strconv.FormatInt(s.Total, 10))
saveJSON("subscriptions", "complete_list", s)
sort.Slice(s.Subscriptions, func(i, j int) bool {
return s.Subscriptions[i].CreatedAt.Before(s.Subscriptions[j].CreatedAt)
})
start := len(s.Subscriptions) - 10
if start < 0 {
start = 0
}
lastSubs := s.Subscriptions[start:]
var lastSubsSlice []string
for _, v := range lastSubs {
lastSubsSlice = append(lastSubsSlice, v.User.DisplayName)
}
saveContent("subscriptions", "last_ten", strings.Join(lastSubsSlice, "\n"))
}

11
user.go
View File

@ -14,9 +14,7 @@ import (
"golang.org/x/oauth2"
)
var (
user *User
)
var user *User
type WebUser struct {
ID string
@ -119,7 +117,7 @@ func getUser(u *User) error {
func saveUser() {
log.Info("Save user")
f, err := os.OpenFile(".user.json", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
f, err := os.OpenFile(".user.json", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600)
if err != nil {
log.WithError(err).Fatal("Unable to open file")
}
@ -155,10 +153,7 @@ func loadUser() {
return
}
if err := getChannel(u); err != nil {
log.WithError(err).Error("Unable to get channel information")
return
}
getChannel(u)
user = u
}