Optimize handling of very large slack teams. Fixes #695

Stop getting users if we reach 2000 users. Slack will rate-limit us
even if we follow their limits.
This means that we now have to lookup every user that says a message
for the first time. This should be less intensive on the API.

This also disables partly fb713ed91bfb48c0037e489f80be85c54e69953a for now.
ChannelMembers will not be filled.

2 files changed, 27 insertions, 11 deletions

diff --git a/bridge/slack/handlers.go b/bridge/slack/handlers.go
index 5e601eaa..e013b830 100644
--- a/bridge/slack/handlers.go
+++ b/bridge/slack/handlers.go
@@ -75,8 +75,6 @@ func (b *Bslack) handleSlackClient(messages chan *config.Message) {
 			// When we join a channel we update the full list of users as
 			// well as the information for the channel that we joined as this
 			// should now tell that we are a member of it.
-			b.populateUsers(false)
-
 			b.channelsMutex.Lock()
 			b.channelsByID[ev.Channel.ID] = &ev.Channel
 			b.channelsByName[ev.Channel.Name] = &ev.Channel
@@ -202,7 +200,6 @@ func (b *Bslack) handleMessageEvent(ev *slack.MessageEvent) (*config.Message, er
 func (b *Bslack) handleStatusEvent(ev *slack.MessageEvent, rmsg *config.Message) bool {
 	switch ev.SubType {
 	case sChannelJoined, sMemberJoined:
-		b.populateUsers(false)
 		// There's no further processing needed on channel events
 		// so we return 'true'.
 		return true
diff --git a/bridge/slack/helpers.go b/bridge/slack/helpers.go
index d84353f0..9959d4fa 100644
--- a/bridge/slack/helpers.go
+++ b/bridge/slack/helpers.go
@@ -13,6 +13,13 @@ import (
 
 func (b *Bslack) getUser(id string) *slack.User {
 	b.usersMutex.RLock()
+	user, ok := b.users[id]
+	b.usersMutex.RUnlock()
+	if ok {
+		return user
+	}
+	b.populateUser(id)
+	b.usersMutex.RLock()
 	defer b.usersMutex.RUnlock()
 
 	return b.users[id]
@@ -102,9 +109,11 @@ func (b *Bslack) populateUsers(wait bool) {
 
 	newUsers := map[string]*slack.User{}
 	pagination := b.sc.GetUsersPaginated(slack.GetUsersOptionLimit(200))
+	count := 0
 	for {
 		var err error
 		pagination, err = pagination.Next(context.Background())
+		time.Sleep(time.Second)
 		if err != nil {
 			if pagination.Done(err) {
 				break
@@ -120,6 +129,13 @@ func (b *Bslack) populateUsers(wait bool) {
 		for i := range pagination.Users {
 			newUsers[pagination.Users[i].ID] = &pagination.Users[i]
 		}
+		b.Log.Debugf("getting %d users", len(pagination.Users))
+		count++
+		// more > 2000 users, slack will complain and ratelimit. break
+		if count > 10 {
+			b.Log.Info("Large slack detected > 2000 users, skipping loading complete userlist.")
+			break
+		}
 	}
 
 	b.usersMutex.Lock()
@@ -172,15 +188,18 @@ func (b *Bslack) populateChannels(wait bool) {
 			newChannelsByID[channels[i].ID] = &channels[i]
 			newChannelsByName[channels[i].Name] = &channels[i]
 			// also find all the members in every channel
-			members, err := b.getUsersInConversation(channels[i].ID)
-			if err != nil {
-				if err = b.handleRateLimit(err); err != nil {
-					b.Log.Errorf("Could not retrieve channel members: %#v", err)
-					return
+			// comment for now, issues on big slacks
+			/*
+				members, err := b.getUsersInConversation(channels[i].ID)
+				if err != nil {
+					if err = b.handleRateLimit(err); err != nil {
+						b.Log.Errorf("Could not retrieve channel members: %#v", err)
+						return
+					}
+					continue
 				}
-				continue
-			}
-			newChannelMembers[channels[i].ID] = members
+				newChannelMembers[channels[i].ID] = members
+			*/
 		}
 
 		if nextCursor == "" {