initial version of collector

This commit is contained in:
Martin Thielecke 2020-05-09 01:51:02 +02:00
parent f2100d8e8b
commit 3c6e3df23c
Signed by: mthie
GPG Key ID: D1D25A85C8604DFB
11 changed files with 798 additions and 0 deletions

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
settings.yml
.user.json

76
channel.go Normal file
View File

@ -0,0 +1,76 @@
package main
import (
"encoding/json"
"net/http"
log "github.com/sirupsen/logrus"
)
type TwitchChannel struct {
ID string `json:"_id"`
BroadcasterLanguage string `json:"broadcaster_language"`
BroadcasterType string `json:"broadcaster_type"`
CreatedAt string `json:"created_at"`
DisplayName string `json:"display_name"`
Email string `json:"email"`
Followers int64 `json:"followers"`
Game string `json:"game"`
Language string `json:"language"`
Logo string `json:"logo"`
Mature bool `json:"mature"`
Name string `json:"name"`
Partner bool `json:"partner"`
ProfileBanner string `json:"profile_banner"`
ProfileBannerBackgroundColor string `json:"profile_banner_background_color"`
Status string `json:"status"`
StreamKey string `json:"stream_key"`
UpdatedAt string `json:"updated_at"`
URL string `json:"url"`
VideoBanner string `json:"video_banner"`
Views int64 `json:"views"`
}
func getChannel(u *User) error {
client := &http.Client{}
req, err := http.NewRequest("GET", "https://api.twitch.tv/kraken/channel", nil)
if err != nil {
log.WithError(err).Error("Unable to create http request to get twitch channel data")
return err
}
req.Header.Set("Client-ID", settings.ClientID)
req.Header.Set("Accept", "application/vnd.twitchtv.v5+json")
req.Header.Set("Authorization", "OAuth "+u.Token.AccessToken)
resp, err := client.Do(req)
if err != nil {
log.WithError(err).Error("Unable to get twitch channel data")
return err
}
defer resp.Body.Close()
c := &TwitchChannel{}
if err := json.NewDecoder(resp.Body).Decode(c); err != nil {
log.WithError(err).Error("Unable to parse twitch channel data")
return err
}
u.TwitchChannel = c
return nil
}
func (c *TwitchChannel) SaveFiles() {
data, err := fieldsToMap(c)
if err != nil {
log.WithError(err).Error("Unable to convert channel to map")
return
}
for k, v := range data {
saveContent("channel", k, v)
}
}

96
follower.go Normal file
View File

@ -0,0 +1,96 @@
package main
import (
"context"
"encoding/json"
"net/http"
"sort"
"strconv"
"strings"
"time"
log "github.com/sirupsen/logrus"
)
type TwitchFollowers struct {
Total int64 `json:"total"`
Data []*TwitchFollower `json:"data"`
Pagination *TwitchPagination `json:"pagination"`
}
type TwitchPagination struct {
Cursor string `json:"cursor"`
}
type TwitchFollower struct {
FromID string `json:"from_id"`
FromName string `json:"from_name"`
ToID string `json:"to_id"`
ToName string `json:"to_name"`
FollowedAt time.Time `json:"followed_at"`
}
func getFollows(u *User) {
result := &TwitchFollowers{}
after := ""
for {
client := twitchOauthConfig.Client(context.Background(), u.Token)
req, err := http.NewRequest("GET", "https://api.twitch.tv/helix/users/follows?to_id="+u.ID+after, nil)
if err != nil {
log.WithError(err).Error("Unable to create http request to get twitch follower data")
return
}
req.Header.Set("Client-ID", settings.ClientID)
resp, err := client.Do(req)
if err != nil {
log.WithError(err).Error("Unable to get twitch follower data")
return
}
t := &TwitchFollowers{}
if err := json.NewDecoder(resp.Body).Decode(&t); err != nil {
log.WithError(err).Error("Unable to parse twitch followers data")
}
resp.Body.Close()
if len(t.Data) == 0 {
break
}
result.Total = t.Total
result.Data = append(result.Data, t.Data...)
if t.Pagination == nil || t.Pagination.Cursor == "" {
break
}
after = "&after=" + t.Pagination.Cursor
}
if len(result.Data) < 1 {
log.Info("No followers")
return
}
u.TwitchFollowers = result
}
func (f *TwitchFollowers) SaveFiles() {
saveContent("followers", "total", strconv.FormatInt(f.Total, 10))
saveJSON("followers", "complete_list", f)
sort.Slice(f.Data, func(i, j int) bool {
return f.Data[i].FollowedAt.Before(f.Data[j].FollowedAt)
})
start := len(f.Data) - 10
if start < 0 {
start = 0
}
lastFollowers := f.Data[start:]
var lastFollowerSlice []string
for _, v := range lastFollowers {
lastFollowerSlice = append(lastFollowerSlice, v.FromName)
}
saveContent("followers", "last_ten", strings.Join(lastFollowerSlice, "\n"))
}

110
main.go Normal file
View File

@ -0,0 +1,110 @@
package main
import (
"context"
"net/http"
"time"
"github.com/Luzifer/rconfig"
gorilla "github.com/gorilla/mux"
log "github.com/sirupsen/logrus"
"golang.org/x/oauth2"
)
var cfg = struct {
SettingsFile string `default:"settings.yml" flag:"settings-file" description:"Path to settings file"`
}{}
func main() {
rconfig.Parse(&cfg)
settingsUpdater()
loadUser()
log.Infof("User: %+v", user)
mux := gorilla.NewRouter()
mux.HandleFunc("/login", handleTwitchLogin)
mux.HandleFunc("/callback", handleTwitchCallback)
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("Hi!"))
})
go func() {
handleSaves()
c := time.Tick(20 * time.Second)
for range c {
handleSaves()
}
}()
log.Info("Starting webserver...")
log.Fatal(http.ListenAndServe(":8083", mux))
}
func handleSaves() {
if user == nil {
return
}
getUser(user)
if user.TwitchUser != nil {
user.TwitchUser.SaveFiles()
}
getChannel(user)
if user.TwitchChannel != nil {
user.TwitchChannel.SaveFiles()
}
getFollows(user)
if user.TwitchFollowers != nil {
user.TwitchFollowers.SaveFiles()
}
getSubs(user)
if user.TwitchSubscriptions != nil {
user.TwitchSubscriptions.SaveFiles()
}
getStreams(user)
if user.TwitchStream != nil {
user.TwitchStream.SaveFiles()
}
}
func handleTwitchLogin(w http.ResponseWriter, r *http.Request) {
url := twitchOauthConfig.AuthCodeURL(settings.VerificationToken, oauth2.AccessTypeOffline)
http.Redirect(w, r, url, http.StatusTemporaryRedirect)
}
func handleTwitchCallback(w http.ResponseWriter, r *http.Request) {
twitchAuthToToken(r.FormValue("state"), r.FormValue("code"), w, r)
http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
}
func twitchAuthToToken(state string, code string, w http.ResponseWriter, r *http.Request) {
if state != settings.VerificationToken {
log.Fatal("invalid oauth state")
}
timeoutCtx, cancel := context.WithTimeout(oauth2.NoContext, 5*time.Second)
defer cancel()
token, err := twitchOauthConfig.Exchange(timeoutCtx, code, oauth2.AccessTypeOffline)
if err != nil {
http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
log.WithError(err).Error("code exchange failed")
return
}
u, err := createUser(token)
if err != nil {
log.WithError(err).Error("Unable to get user")
return
}
u.Token = token
user = u
log.Infof("User: %+v, User: %+v", user, user.TwitchUser)
saveUser()
}

65
reflect.go Normal file
View File

@ -0,0 +1,65 @@
package main
import (
"reflect"
"strconv"
"strings"
"time"
"github.com/pkg/errors"
)
func fieldsToMap(in interface{}) (map[string]string, error) {
if reflect.TypeOf(in).Kind() != reflect.Ptr {
return nil, errors.New("Non-pointer given")
}
if kind := reflect.ValueOf(in).Elem().Kind(); kind != reflect.Struct {
return nil, errors.Errorf("Non-struct given: %s", kind)
}
var out = map[string]string{}
st := reflect.ValueOf(in).Elem()
for i := 0; i < st.NumField(); i++ {
valField := st.Field(i)
typeField := st.Type().Field(i)
jsonTag := strings.Split(typeField.Tag.Get("json"), ",")[0]
if jsonTag == "" {
// Empty tag, skip
continue
}
switch typeField.Type {
case reflect.TypeOf(time.Time{}):
out[jsonTag] = valField.Addr().Interface().(*time.Time).String()
continue
}
switch typeField.Type.Kind() {
case reflect.Bool:
out[jsonTag] = strconv.FormatBool(valField.Bool())
case reflect.Int, reflect.Int64:
out[jsonTag] = strconv.FormatInt(valField.Int(), 10)
case reflect.String:
out[jsonTag] = valField.String()
case reflect.Slice:
switch typeField.Type.Elem().Kind() {
case reflect.String:
res := valField.Addr().Interface().(*[]string)
out[jsonTag] = strings.Join(*res, "\n")
}
default:
return nil, errors.Errorf("Unhandled field type: %s", typeField.Type.String())
}
}
return out, nil
}

42
save.go Normal file
View File

@ -0,0 +1,42 @@
package main
import (
"encoding/json"
"io/ioutil"
"os"
"path"
log "github.com/sirupsen/logrus"
)
func saveContent(kind, filename, content string) {
p := path.Join(kind, filename+".txt")
if err := os.MkdirAll(kind, 0777); err != nil {
log.WithField("path", p).WithError(err).Error("Unable to create directory")
return
}
if err := ioutil.WriteFile(p, []byte(content), 0777); err != nil {
log.WithError(err).Error("Unable to write content")
}
}
func saveJSON(kind, filename string, data interface{}) {
p := path.Join(kind, filename+".json")
if err := os.MkdirAll(kind, 0777); err != nil {
log.WithField("path", p).WithError(err).Error("Unable to create directory")
return
}
f, err := os.Create(p)
if err != nil {
log.WithField("path", p).WithError(err).Error("Unable to create file")
return
}
defer f.Close()
if err := json.NewEncoder(f).Encode(data); err != nil {
log.WithField("path", p).WithError(err).Error("Unable to encode json")
return
}
}

60
settings.go Normal file
View File

@ -0,0 +1,60 @@
package main
import (
"bytes"
"io/ioutil"
"time"
log "github.com/sirupsen/logrus"
"golang.org/x/oauth2"
"golang.org/x/oauth2/twitch"
"gopkg.in/yaml.v2"
)
var (
settings *Settings
twitchOauthConfig *oauth2.Config
)
type Settings struct {
ClientID string `yaml:"client_id"`
ClientSecret string `yaml:"client_secret"`
VerificationToken string `yaml:"verification_token"`
}
func loadSettings() {
data, err := ioutil.ReadFile(cfg.SettingsFile)
if err != nil {
log.WithError(err).Errorf("Unable to read %s", cfg.SettingsFile)
return
}
s := &Settings{}
b := bytes.NewBuffer(data)
if err := yaml.NewDecoder(b).Decode(s); err != nil {
log.WithError(err).Errorf("Unable to decode %s", cfg.SettingsFile)
return
}
settings = s
twitchOauthConfig = &oauth2.Config{
RedirectURL: "http://localhost:8080/callback",
ClientID: settings.ClientID,
ClientSecret: settings.ClientSecret,
Scopes: []string{"channel:read:subscriptions", "user:read:broadcast", "chat:read", "chat:edit", "channel_read", "channel_editor", "channel_subscriptions", "channel:moderate", "bits:read", "channel:read:redemptions"},
Endpoint: twitch.Endpoint,
}
}
func settingsUpdater() {
loadSettings()
go func() {
c := time.Tick(time.Minute)
for range c {
loadSettings()
}
}()
}

3
settings.yml.dist Normal file
View File

@ -0,0 +1,3 @@
client_id: "your twitch client id"
client_secret: "your twitch client secret"
verification_token: "a random string!"

87
stream.go Normal file
View File

@ -0,0 +1,87 @@
package main
import (
"context"
"encoding/json"
"net/http"
"time"
log "github.com/sirupsen/logrus"
)
type TwitchStreams struct {
Data []*TwitchStream `json:"data"`
Pagination *TwitchPagination `json:"pagination"`
}
type TwitchStream struct {
GameID string `json:"game_id"`
ID string `json:"id"`
Language string `json:"language"`
StartedAt time.Time `json:"started_at"`
TagIDs []string `json:"tag_ids"`
ThumbnailURL string `json:"thumbnail_url"`
Title string `json:"title"`
Type string `json:"type"`
UserID string `json:"user_id"`
UserName string `json:"user_name"`
ViewerCount int `json:"viewer_count"`
}
func getStreams(u *User) {
result := &TwitchStreams{}
after := ""
for {
client := twitchOauthConfig.Client(context.Background(), u.Token)
req, err := http.NewRequest("GET", "https://api.twitch.tv/helix/streams?user_id="+u.ID+after, nil)
if err != nil {
log.WithError(err).Error("Unable to create http request to get twitch streams data")
return
}
req.Header.Set("Client-ID", settings.ClientID)
resp, err := client.Do(req)
if err != nil {
log.WithError(err).Error("Unable to get twitch stream data")
return
}
t := &TwitchStreams{}
if err := json.NewDecoder(resp.Body).Decode(&t); err != nil {
log.WithError(err).Error("Unable to parse twitch streams data")
}
resp.Body.Close()
if len(t.Data) == 0 {
break
}
result.Data = append(result.Data, t.Data...)
if t.Pagination == nil || t.Pagination.Cursor == "" {
break
}
after = "&after=" + t.Pagination.Cursor
}
if len(result.Data) < 1 {
log.Info("No streams")
return
}
if len(result.Data) > 0 {
u.TwitchStream = result.Data[0]
}
}
func (s *TwitchStream) SaveFiles() {
data, err := fieldsToMap(s)
if err != nil {
log.WithError(err).Error("Unable to convert stream to map")
return
}
for k, v := range data {
saveContent("stream", k, v)
}
}

106
subs.go Normal file
View File

@ -0,0 +1,106 @@
package main
import (
"encoding/json"
"fmt"
"net/http"
"sort"
"strconv"
"strings"
"time"
log "github.com/sirupsen/logrus"
)
type TwitchSubscriptions struct {
Total int64 `json:"_total"`
Subscriptions []*TwitchSubscription `json:"subscriptions"`
}
type TwitchSubscription struct {
ID string `json:"_id"`
CreatedAt time.Time `json:"created_at"`
IsGift bool `json:"is_gift"`
SubPlan string `json:"sub_plan"`
SubPlanName string `json:"sub_plan_name"`
User *TwitchSubUser `json:"user"`
}
type TwitchSubUser struct {
ID string `json:"_id"`
Bio string `json:"bio"`
CreatedAt time.Time `json:"created_at"`
DisplayName string `json:"display_name"`
Logo string `json:"logo"`
Name string `json:"name"`
Type string `json:"type"`
UpdatedAt time.Time `json:"updated_at"`
}
func getSubs(u *User) {
result := &TwitchSubscriptions{}
limit := 100
offset := 0
for {
client := http.Client{}
req, err := http.NewRequest("GET", fmt.Sprintf("https://api.twitch.tv/kraken/channels/%s/subscriptions?limit=%d&offset=%d", u.TwitchChannel.ID, limit, offset), nil)
if err != nil {
log.WithError(err).Error("Unable to create http request to get twitch subs data")
return
}
req.Header.Set("Client-ID", settings.ClientID)
req.Header.Set("Accept", "application/vnd.twitchtv.v5+json")
req.Header.Set("Authorization", "OAuth "+u.Token.AccessToken)
resp, err := client.Do(req)
if err != nil {
log.WithError(err).Error("Unable to get twitch subs data")
return
}
t := &TwitchSubscriptions{}
if err := json.NewDecoder(resp.Body).Decode(&t); err != nil {
log.WithError(err).Error("Unable to parse twitch followers data")
}
resp.Body.Close()
if len(t.Subscriptions) == 0 {
break
}
result.Total = t.Total
result.Subscriptions = append(result.Subscriptions, t.Subscriptions...)
offset += limit
}
if len(result.Subscriptions) < 1 {
log.Info("No Subs")
return
}
u.TwitchSubscriptions = result
}
func (s *TwitchSubscriptions) SaveFiles() {
saveContent("subscriptions", "total", strconv.FormatInt(s.Total, 10))
saveJSON("subscriptions", "complete_list", s)
sort.Slice(s.Subscriptions, func(i, j int) bool {
return s.Subscriptions[i].CreatedAt.Before(s.Subscriptions[j].CreatedAt)
})
start := len(s.Subscriptions) - 10
if start < 0 {
start = 0
}
lastSubs := s.Subscriptions[start:]
var lastSubsSlice []string
for _, v := range lastSubs {
lastSubsSlice = append(lastSubsSlice, v.User.DisplayName)
}
saveContent("subscriptions", "last_ten", strings.Join(lastSubsSlice, "\n"))
}

151
user.go Normal file
View File

@ -0,0 +1,151 @@
package main
import (
"bytes"
"context"
"encoding/json"
"io/ioutil"
"net/http"
"os"
"github.com/go-irc/irc"
log "github.com/sirupsen/logrus"
"golang.org/x/oauth2"
)
var (
user *User
)
type User struct {
ID string
Name string
DisplayName string
Token *oauth2.Token
IRCClient *irc.Client `json:"-"`
TwitchUser *TwitchUser `json:"-"`
TwitchChannel *TwitchChannel `json:"-"`
TwitchFollowers *TwitchFollowers `json:"-"`
TwitchSubscriptions *TwitchSubscriptions `json:"-"`
TwitchStream *TwitchStream `json:"-"`
}
type TwitchUser struct {
ID string `json:"id"`
Login string `json:"login"`
DisplayName string `json:"display_name"`
Type string `json:"type"`
BroadcasterType string `json:"broadcaster_type"`
Description string `json:"description"`
ProfileImage string `json:"profile_image_url"`
OfflineImageURL string `json:"offline_image_url"`
ViewCount int64 `json:"view_count"`
}
func createUser(token *oauth2.Token) (*User, error) {
u := &User{
Token: token,
}
if err := getUser(u); err != nil {
log.WithError(err).Error("Unable to get user")
return nil, err
}
u.ID = u.TwitchUser.ID
u.Name = u.TwitchUser.Login
u.DisplayName = u.TwitchUser.DisplayName
return u, nil
}
func getUser(u *User) error {
client := twitchOauthConfig.Client(context.Background(), u.Token)
req, err := http.NewRequest("GET", "https://api.twitch.tv/helix/users", nil)
if err != nil {
log.WithError(err).Error("Unable to create http request to get twitch user data")
return err
}
req.Header.Set("Client-ID", settings.ClientID)
resp, err := client.Do(req)
if err != nil {
log.WithError(err).Error("Unable to get twitch user data")
return err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.WithError(err).Error("Unable to get response body for user data")
return err
}
users := map[string][]*TwitchUser{}
if err := json.Unmarshal(b, &users); err != nil {
log.WithError(err).Error("Unable to parse twitch user data")
return err
}
u.TwitchUser = users["data"][0]
return nil
}
func saveUser() {
log.Info("Save user")
f, err := os.OpenFile(".user.json", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
log.WithError(err).Fatal("Unable to open file")
}
defer f.Close()
if err := json.NewEncoder(f).Encode(user); err != nil {
log.WithError(err).Fatal("Unable to encode users data")
}
}
func loadUser() {
data, err := ioutil.ReadFile(".user.json")
if err != nil {
log.WithError(err).Error("Unable to read .users.json")
return
}
var u *User
b := bytes.NewBuffer(data)
if err := json.NewDecoder(b).Decode(&u); err != nil {
log.WithError(err).Error("Unable to decode .users.json")
return
}
if u == nil {
log.Warning("User not existent")
return
}
if err := getUser(u); err != nil {
log.WithError(err).Error("Unable to get user information")
return
}
if err := getChannel(u); err != nil {
log.WithError(err).Error("Unable to get channel information")
return
}
user = u
}
func (t *TwitchUser) SaveFiles() {
data, err := fieldsToMap(t)
if err != nil {
log.WithError(err).Error("Unable to convert user to map")
return
}
for k, v := range data {
saveContent("user", k, v)
}
}