summaryrefslogtreecommitdiffstats
path: root/bridge/slack
diff options
context:
space:
mode:
authorWim <wim@42.be>2019-01-30 23:26:36 +0100
committerWim <wim@42.be>2019-01-30 23:28:37 +0100
commit4ae028fe73c9b5217b3b4a403432d18bee18a057 (patch)
tree5380a88faf2d208c4a31296ee2a8c33effc12cfc /bridge/slack
parent707db950c83e7fe2578ec206c6eeb330cc2f5f5b (diff)
downloadmatterbridge-msglm-4ae028fe73c9b5217b3b4a403432d18bee18a057.tar.gz
matterbridge-msglm-4ae028fe73c9b5217b3b4a403432d18bee18a057.tar.bz2
matterbridge-msglm-4ae028fe73c9b5217b3b4a403432d18bee18a057.zip
Optimize handling of very large slack teams. Fixes #695
Stop getting users if we reach 2000 users. Slack will rate-limit us even if we follow their limits. This means that we now have to lookup every user that says a message for the first time. This should be less intensive on the API. This also disables partly fb713ed91bfb48c0037e489f80be85c54e69953a for now. ChannelMembers will not be filled.
Diffstat (limited to 'bridge/slack')
-rw-r--r--bridge/slack/handlers.go3
-rw-r--r--bridge/slack/helpers.go35
2 files changed, 27 insertions, 11 deletions
diff --git a/bridge/slack/handlers.go b/bridge/slack/handlers.go
index 5e601eaa..e013b830 100644
--- a/bridge/slack/handlers.go
+++ b/bridge/slack/handlers.go
@@ -75,8 +75,6 @@ func (b *Bslack) handleSlackClient(messages chan *config.Message) {
// When we join a channel we update the full list of users as
// well as the information for the channel that we joined as this
// should now tell that we are a member of it.
- b.populateUsers(false)
-
b.channelsMutex.Lock()
b.channelsByID[ev.Channel.ID] = &ev.Channel
b.channelsByName[ev.Channel.Name] = &ev.Channel
@@ -202,7 +200,6 @@ func (b *Bslack) handleMessageEvent(ev *slack.MessageEvent) (*config.Message, er
func (b *Bslack) handleStatusEvent(ev *slack.MessageEvent, rmsg *config.Message) bool {
switch ev.SubType {
case sChannelJoined, sMemberJoined:
- b.populateUsers(false)
// There's no further processing needed on channel events
// so we return 'true'.
return true
diff --git a/bridge/slack/helpers.go b/bridge/slack/helpers.go
index d84353f0..9959d4fa 100644
--- a/bridge/slack/helpers.go
+++ b/bridge/slack/helpers.go
@@ -13,6 +13,13 @@ import (
func (b *Bslack) getUser(id string) *slack.User {
b.usersMutex.RLock()
+ user, ok := b.users[id]
+ b.usersMutex.RUnlock()
+ if ok {
+ return user
+ }
+ b.populateUser(id)
+ b.usersMutex.RLock()
defer b.usersMutex.RUnlock()
return b.users[id]
@@ -102,9 +109,11 @@ func (b *Bslack) populateUsers(wait bool) {
newUsers := map[string]*slack.User{}
pagination := b.sc.GetUsersPaginated(slack.GetUsersOptionLimit(200))
+ count := 0
for {
var err error
pagination, err = pagination.Next(context.Background())
+ time.Sleep(time.Second)
if err != nil {
if pagination.Done(err) {
break
@@ -120,6 +129,13 @@ func (b *Bslack) populateUsers(wait bool) {
for i := range pagination.Users {
newUsers[pagination.Users[i].ID] = &pagination.Users[i]
}
+ b.Log.Debugf("getting %d users", len(pagination.Users))
+ count++
+ // more > 2000 users, slack will complain and ratelimit. break
+ if count > 10 {
+ b.Log.Info("Large slack detected > 2000 users, skipping loading complete userlist.")
+ break
+ }
}
b.usersMutex.Lock()
@@ -172,15 +188,18 @@ func (b *Bslack) populateChannels(wait bool) {
newChannelsByID[channels[i].ID] = &channels[i]
newChannelsByName[channels[i].Name] = &channels[i]
// also find all the members in every channel
- members, err := b.getUsersInConversation(channels[i].ID)
- if err != nil {
- if err = b.handleRateLimit(err); err != nil {
- b.Log.Errorf("Could not retrieve channel members: %#v", err)
- return
+ // comment for now, issues on big slacks
+ /*
+ members, err := b.getUsersInConversation(channels[i].ID)
+ if err != nil {
+ if err = b.handleRateLimit(err); err != nil {
+ b.Log.Errorf("Could not retrieve channel members: %#v", err)
+ return
+ }
+ continue
}
- continue
- }
- newChannelMembers[channels[i].ID] = members
+ newChannelMembers[channels[i].ID] = members
+ */
}
if nextCursor == "" {