mthie
/
twitch_data_collector
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

fix twitch APIs and usage values

Browse Source
master
Martin Thielecke 5 months ago
parent 780d258cb4
commit 1bb87788f8
Signed by: mthie
GPG Key ID: D1D25A85C8604DFB
5 changed files with 138 additions and 152 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 101
      channel.go
  2. 49
      follower.go
  3. 37
      main.go
  4. 92
      subs.go
  5. 11
      user.go

101
channel.go
Unescape View File

@ -1,72 +1,73 @@
package main
import (
"context"
"encoding/json"
"io/ioutil"
"net/http"
log "github.com/sirupsen/logrus"
)
type TwitchChannel struct {
ID string `json:"_id"`
type TwitchChannels struct {
Data []*TwitchChannel `json:"data"`
Pagination *TwitchPagination `json:"pagination"`
}
BroadcasterLanguage string `json:"broadcaster_language"`
BroadcasterType string `json:"broadcaster_type"`
CreatedAt string `json:"created_at"`
DisplayName string `json:"display_name"`
Email string `json:"email"`
Followers int64 `json:"followers"`
Game string `json:"game"`
Language string `json:"language"`
Logo string `json:"logo"`
Mature bool `json:"mature"`
Name string `json:"name"`
Partner bool `json:"partner"`
ProfileBanner string `json:"profile_banner"`
ProfileBannerBackgroundColor string `json:"profile_banner_background_color"`
Status string `json:"status"`
StreamKey string `json:"stream_key"`
UpdatedAt string `json:"updated_at"`
URL string `json:"url"`
VideoBanner string `json:"video_banner"`
Views int64 `json:"views"`
type TwitchChannel struct {
BroadcasterID string `json:"broadcaster_id"`
BroadcasterLogin string `json:"broadcaster_login"`
BroadcasterName string `json:"broadcaster_name"`
BroadcasterLanguage string `json:"broadcaster_language"`
GameID string `json:"game_id"`
GameName string `json:"game_name"`
Title string `json:"title"`
Delay int `json:"delay"`
}
func getChannel(u *User) error {
client := &http.Client{}
req, err := http.NewRequest("GET", "https://api.twitch.tv/kraken/channel", nil)
if err != nil {
log.WithError(err).Error("Unable to create http request to get twitch channel data")
return err
}
func getChannel(u *User) {
result := &TwitchChannels{}
req.Header.Set("Client-ID", settings.ClientID)
req.Header.Set("Accept", "application/vnd.twitchtv.v5+json")
req.Header.Set("Authorization", "OAuth "+u.Token.AccessToken)
after := ""
for {
client := twitchOauthConfig.Client(context.Background(), u.Token)
req, err := http.NewRequest("GET", "https://api.twitch.tv/helix/channels?broadcaster_id="+u.ID+after, nil)
if err != nil {
log.WithError(err).Error("Unable to create http request to get twitch streams data")
return
}
req.Header.Set("Client-ID", settings.ClientID)
resp, err := client.Do(req)
if err != nil {
log.WithError(err).Error("Unable to get twitch channel data")
return err
}
resp, err := client.Do(req)
if err != nil {
log.WithError(err).Error("Unable to get twitch stream data")
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.WithError(err).Error("Unable to read response")
}
c := &TwitchChannel{}
t := &TwitchChannels{}
if err := json.NewDecoder(resp.Body).Decode(&t); err != nil {
log.WithError(err).Error("Unable to parse twitch streams data")
}
resp.Body.Close()
if len(t.Data) == 0 {
break
}
if err := json.Unmarshal(b, c); err != nil {
log.WithError(err).Error("Unable to parse twitch channel data")
log.Infof("Data: %s", string(b))
return err
result.Data = append(result.Data, t.Data...)
if t.Pagination == nil || t.Pagination.Cursor == "" {
break
}
after = "&after=" + t.Pagination.Cursor
}
u.TwitchChannel = c
if len(result.Data) < 1 {
log.Info("No streams")
return
}
return nil
if len(result.Data) > 0 {
u.TwitchChannel = result.Data[0]
}
}
func (c *TwitchChannel) SaveFiles() {

49
follower.go
Unescape View File

@ -1,13 +1,10 @@
package main
import (
"bytes"
"context"
"encoding/json"
"io/ioutil"
"net/http"
"os"
"strings"
"strconv"
"time"
log "github.com/sirupsen/logrus"
@ -81,36 +78,22 @@ func getFollows(u *User, max int) {
u.TwitchFollowers = result
}
func readActiveUsers() map[string]bool {
res, err := ioutil.ReadFile(os.Args[1])
if err != nil {
log.WithError(err).Fatal("Unable to read active users")
}
data := make(map[string]bool)
if err := json.NewDecoder(bytes.NewBuffer(res)).Decode(&data); err != nil {
log.WithError(err).Fatal("Unable to decode json")
}
return data
}
func (f *TwitchFollowers) SaveFiles() {
fs := []string{}
active := readActiveUsers()
ids := []string{}
for _, follower := range f.Data {
name := strings.ToLower(follower.FromName)
id := follower.FromID
if _, ok := active[name]; ok {
continue
saveContent("followers", "total", strconv.FormatInt(f.Total, 10))
saveJSON("followers", "complete_list", f)
/*
fs := []string{}
ids := []string{}
for _, follower := range f.Data {
name := strings.ToLower(follower.FromName)
id := follower.FromID
fs = append(fs, name)
ids = append(ids, id)
}
fs = append(fs, name)
ids = append(ids, id)
}
saveJSON("followers", "inactive", fs)
saveContent("followers", "inactive_ids", strings.Join(ids, "\n"))
saveJSON("followers", "inactive", fs)
saveContent("followers", "inactive_ids", strings.Join(ids, "\n"))
*/
}

37
main.go
Unescape View File

@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"net/http"
"os"
"time"
"github.com/Luzifer/rconfig"
@ -49,7 +48,18 @@ func main() {
go func() {
if user != nil {
handleSaves()
os.Exit(0)
}
interval := 20 * time.Second
if settings.UpdateInterval > 10*time.Second {
interval = settings.UpdateInterval
}
log.Infof("Update interval: %s", interval)
c := time.Tick(interval)
for range c {
if user == nil {
continue
}
handleSaves()
}
}()
@ -63,16 +73,39 @@ func handleSaves() {
}
getUser(user)
if user.TwitchUser != nil {
user.TwitchUser.SaveFiles()
}
getChannel(user)
if user.TwitchChannel != nil {
user.TwitchChannel.SaveFiles()
}
maxFollower := -1
if settings.MaxFollowers > 0 {
maxFollower = settings.MaxFollowers
}
log.Infof("Fetch max follower: %d", maxFollower)
getFollows(user, maxFollower)
if user.TwitchFollowers != nil {
user.TwitchFollowers.SaveFiles()
}
maxSubs := -1
if settings.MaxSubs > 0 {
maxSubs = settings.MaxSubs
}
log.Infof("Fetch max subs: %d", maxSubs)
getSubs(user, maxSubs)
if user.TwitchSubscriptions != nil {
user.TwitchSubscriptions.SaveFiles()
}
getStreams(user)
if user.TwitchStream != nil {
user.TwitchStream.SaveFiles()
}
}
func handleTwitchLogin(w http.ResponseWriter, r *http.Request) {

92
subs.go
Unescape View File

@ -1,109 +1,83 @@
package main
import (
"context"
"encoding/json"
"fmt"
"net/http"
"sort"
"strconv"
"strings"
"time"
log "github.com/sirupsen/logrus"
)
type TwitchSubscriptions struct {
Total int64 `json:"_total"`
Subscriptions []*TwitchSubscription `json:"subscriptions"`
Data []*TwitchSubscription `json:"data"`
Pagination *TwitchPagination `json:"pagination"`
Total int64 `json:"total"`
Points int64 `json:"points"`
}
type TwitchSubscription struct {
ID string `json:"_id"`
CreatedAt time.Time `json:"created_at"`
IsGift bool `json:"is_gift"`
SubPlan string `json:"sub_plan"`
SubPlanName string `json:"sub_plan_name"`
User *TwitchSubUser `json:"user"`
}
type TwitchSubUser struct {
ID string `json:"_id"`
Bio string `json:"bio"`
CreatedAt time.Time `json:"created_at"`
DisplayName string `json:"display_name"`
Logo string `json:"logo"`
Name string `json:"name"`
Type string `json:"type"`
UpdatedAt time.Time `json:"updated_at"`
BroadcasterID string `json:"broadcaster_id"`
BroadcasterLogin string `json:"broadcaster_login"`
BroadcasterName string `json:"broadcaster_name"`
GifterID string `json:"gifter_id"`
GifterLogin string `json:"gifter_login"`
GifterName string `json:"gifter_name"`
IsGift bool `json:"is_gift"`
Tier string `json:"gift"`
PlanName string `json:"plan_name"`
UserID string `json:"user_id"`
UserLogin string `json:"user_login"`
UserName string `json:"user_name"`
}
func getSubs(u *User, max int) {
result := &TwitchSubscriptions{}
limit := 100
offset := 0
after := ""
for {
client := http.Client{}
req, err := http.NewRequest("GET", fmt.Sprintf("https://api.twitch.tv/kraken/channels/%s/subscriptions?limit=%d&offset=%d", u.TwitchChannel.ID, limit, offset), nil)
client := twitchOauthConfig.Client(context.Background(), u.Token)
req, err := http.NewRequest("GET", "https://api.twitch.tv/helix/subscriptions?broadcaster_id="+u.ID+after, nil)
if err != nil {
log.WithError(err).Error("Unable to create http request to get twitch subs data")
log.WithError(err).Error("Unable to create http request to get twitch streams data")
return
}
req.Header.Set("Client-ID", settings.ClientID)
req.Header.Set("Accept", "application/vnd.twitchtv.v5+json")
req.Header.Set("Authorization", "OAuth "+u.Token.AccessToken)
resp, err := client.Do(req)
if err != nil {
log.WithError(err).Error("Unable to get twitch subs data")
log.WithError(err).Error("Unable to get twitch stream data")
return
}
t := &TwitchSubscriptions{}
if err := json.NewDecoder(resp.Body).Decode(&t); err != nil {
log.WithError(err).Error("Unable to parse twitch followers data")
log.WithError(err).Error("Unable to parse twitch streams data")
}
resp.Body.Close()
if len(t.Subscriptions) == 0 {
if len(t.Data) == 0 {
break
}
result.Total = t.Total
result.Subscriptions = append(result.Subscriptions, t.Subscriptions...)
if max > -1 && len(result.Subscriptions) >= max {
result.Data = append(result.Data, t.Data...)
if t.Pagination == nil || t.Pagination.Cursor == "" {
break
}
offset += limit
after = "&after=" + t.Pagination.Cursor
}
if len(result.Subscriptions) < 1 {
log.Info("No Subs")
if len(result.Data) < 1 {
log.Info("No streams")
return
}
u.TwitchSubscriptions = result
if len(result.Data) > 0 {
u.TwitchSubscriptions = result
}
}
func (s *TwitchSubscriptions) SaveFiles() {
saveContent("subscriptions", "total", strconv.FormatInt(s.Total, 10))
saveJSON("subscriptions", "complete_list", s)
sort.Slice(s.Subscriptions, func(i, j int) bool {
return s.Subscriptions[i].CreatedAt.Before(s.Subscriptions[j].CreatedAt)
})
start := len(s.Subscriptions) - 10
if start < 0 {
start = 0
}
lastSubs := s.Subscriptions[start:]
var lastSubsSlice []string
for _, v := range lastSubs {
lastSubsSlice = append(lastSubsSlice, v.User.DisplayName)
}
saveContent("subscriptions", "last_ten", strings.Join(lastSubsSlice, "\n"))
}

11
user.go
Unescape View File

@ -14,9 +14,7 @@ import (
"golang.org/x/oauth2"
)
var (
user *User
)
var user *User
type WebUser struct {
ID string
@ -119,7 +117,7 @@ func getUser(u *User) error {
func saveUser() {
log.Info("Save user")
f, err := os.OpenFile(".user.json", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
f, err := os.OpenFile(".user.json", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600)
if err != nil {
log.WithError(err).Fatal("Unable to open file")
}
@ -155,10 +153,7 @@ func loadUser() {
return
}
if err := getChannel(u); err != nil {
log.WithError(err).Error("Unable to get channel information")
return
}
getChannel(u)
user = u
}

Write Preview
Loading…
Cancel
Save