From 60cb47827e39550c170681bf152e956b9cdbcd38 Mon Sep 17 00:00:00 2001
From: Eero af Heurlin <eero.afheurlin@iki.fi>
Date: Mon, 29 May 2017 23:39:59 +0300
Subject: [PATCH 01/17] startapp stub

---
 project/slacksync/__init__.py            | 0
 project/slacksync/admin.py               | 3 +++
 project/slacksync/migrations/__init__.py | 0
 project/slacksync/models.py              | 3 +++
 project/slacksync/tests.py               | 3 +++
 project/slacksync/views.py               | 3 +++
 6 files changed, 12 insertions(+)
 create mode 100644 project/slacksync/__init__.py
 create mode 100644 project/slacksync/admin.py
 create mode 100644 project/slacksync/migrations/__init__.py
 create mode 100644 project/slacksync/models.py
 create mode 100644 project/slacksync/tests.py
 create mode 100644 project/slacksync/views.py

diff --git a/project/slacksync/__init__.py b/project/slacksync/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/project/slacksync/admin.py b/project/slacksync/admin.py
new file mode 100644
index 00000000..8c38f3f3
--- /dev/null
+++ b/project/slacksync/admin.py
@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.
diff --git a/project/slacksync/migrations/__init__.py b/project/slacksync/migrations/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/project/slacksync/models.py b/project/slacksync/models.py
new file mode 100644
index 00000000..71a83623
--- /dev/null
+++ b/project/slacksync/models.py
@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.
diff --git a/project/slacksync/tests.py b/project/slacksync/tests.py
new file mode 100644
index 00000000..7ce503c2
--- /dev/null
+++ b/project/slacksync/tests.py
@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.
diff --git a/project/slacksync/views.py b/project/slacksync/views.py
new file mode 100644
index 00000000..91ea44a2
--- /dev/null
+++ b/project/slacksync/views.py
@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.

From 8ca0e4d1f92e221848e0078c560a41bf8ba81255 Mon Sep 17 00:00:00 2001
From: Eero af Heurlin <eero.afheurlin@iki.fi>
Date: Mon, 29 May 2017 23:46:44 +0300
Subject: [PATCH 02/17] add stub management command

---
 project/slacksync/management/__init__.py             |  1 +
 project/slacksync/management/commands/__init__.py    |  1 +
 .../management/commands/sync_slack_users.py          | 12 ++++++++++++
 3 files changed, 14 insertions(+)
 create mode 100644 project/slacksync/management/__init__.py
 create mode 100644 project/slacksync/management/commands/__init__.py
 create mode 100644 project/slacksync/management/commands/sync_slack_users.py

diff --git a/project/slacksync/management/__init__.py b/project/slacksync/management/__init__.py
new file mode 100644
index 00000000..40a96afc
--- /dev/null
+++ b/project/slacksync/management/__init__.py
@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-
diff --git a/project/slacksync/management/commands/__init__.py b/project/slacksync/management/commands/__init__.py
new file mode 100644
index 00000000..40a96afc
--- /dev/null
+++ b/project/slacksync/management/commands/__init__.py
@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-
diff --git a/project/slacksync/management/commands/sync_slack_users.py b/project/slacksync/management/commands/sync_slack_users.py
new file mode 100644
index 00000000..5ddf1d01
--- /dev/null
+++ b/project/slacksync/management/commands/sync_slack_users.py
@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+from django.core.management.base import BaseCommand, CommandError
+
+
+class Command(BaseCommand):
+    help = 'Make sure all members are in Slack and optionally kick non-members'
+
+    def add_arguments(self, parser):
+        pass
+
+    def handle(self, *args, **options):
+        raise NotImplemented()

From c2d434d46e9248cf29a6e2d4d83e8c913c5854e0 Mon Sep 17 00:00:00 2001
From: Eero af Heurlin <eero.afheurlin@iki.fi>
Date: Mon, 29 May 2017 23:47:01 +0300
Subject: [PATCH 03/17] Add new app to local_apps, add required library

---
 project/config/settings/common.py | 1 +
 project/requirements/base.txt     | 1 +
 2 files changed, 2 insertions(+)

diff --git a/project/config/settings/common.py b/project/config/settings/common.py
index 7a3a745f..7b7a21e7 100644
--- a/project/config/settings/common.py
+++ b/project/config/settings/common.py
@@ -65,6 +65,7 @@
     'ndaparser',
     'holviapp',
     'velkoja',
+    'slacksync',
 )
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
diff --git a/project/requirements/base.txt b/project/requirements/base.txt
index f88843f7..3727a5bc 100644
--- a/project/requirements/base.txt
+++ b/project/requirements/base.txt
@@ -41,3 +41,4 @@ django-markdown==0.8.4
 django-settings-export==1.2.1
 holviapi==0.3.20171118
 python-dateutil==2.6.0
+slacker==0.9.50

From 77fd3476128b9ec454df207ac527ca146c90f4f1 Mon Sep 17 00:00:00 2001
From: Eero af Heurlin <eero.afheurlin@iki.fi>
Date: Tue, 30 May 2017 00:33:29 +0300
Subject: [PATCH 04/17] Fetch the apikey from env and util functions to get
 client

---
 project/config/settings/common.py |  1 +
 project/slacksync/utils.py        | 11 +++++++++++
 2 files changed, 12 insertions(+)
 create mode 100644 project/slacksync/utils.py

diff --git a/project/config/settings/common.py b/project/config/settings/common.py
index 7b7a21e7..64d12a58 100644
--- a/project/config/settings/common.py
+++ b/project/config/settings/common.py
@@ -271,6 +271,7 @@
 HOLVI_APIKEY = env('HOLVI_APIKEY', default=None)
 HOLVI_BARCODE_IBAN = env('HOLVI_BARCODE_IBAN', default=None)
 HOLVI_NOTIFICATION_INTERVAL_DAYS = env('HOLVI_NOTIFICATION_INTERVAL_DAYS', default=7)
+SLACK_APIKEY = env('SLACK_APIKEY', default=None)
 
 REST_FRAMEWORK = {
     'DEFAULT_AUTHENTICATION_CLASSES': [
diff --git a/project/slacksync/utils.py b/project/slacksync/utils.py
new file mode 100644
index 00000000..e31203db
--- /dev/null
+++ b/project/slacksync/utils.py
@@ -0,0 +1,11 @@
+from django.conf import settings
+from slacker import Slacker
+
+
+def api_configured():
+    return bool(settings.SLACK_APIKEY)
+
+def get_client(**kwargs):
+    if not api_configured():
+        return False
+    return Slacker(settings.SLACK_APIKEY, **kwargs)

From 86b50a5e045fbfbf25b575df7313a5fc2c617e88 Mon Sep 17 00:00:00 2001
From: Eero af Heurlin <eero.afheurlin@iki.fi>
Date: Tue, 30 May 2017 01:08:41 +0300
Subject: [PATCH 05/17] Working on member sync

---
 project/config/settings/common.py |  1 +
 project/slacksync/__init__.py     |  1 +
 project/slacksync/admin.py        |  1 +
 project/slacksync/membersync.py   | 39 +++++++++++++++++++++++++++++++
 project/slacksync/models.py       |  1 +
 project/slacksync/tests.py        |  1 +
 project/slacksync/utils.py        |  4 +++-
 project/slacksync/views.py        |  1 +
 8 files changed, 48 insertions(+), 1 deletion(-)
 create mode 100644 project/slacksync/membersync.py

diff --git a/project/config/settings/common.py b/project/config/settings/common.py
index 64d12a58..aef34dbf 100644
--- a/project/config/settings/common.py
+++ b/project/config/settings/common.py
@@ -272,6 +272,7 @@
 HOLVI_BARCODE_IBAN = env('HOLVI_BARCODE_IBAN', default=None)
 HOLVI_NOTIFICATION_INTERVAL_DAYS = env('HOLVI_NOTIFICATION_INTERVAL_DAYS', default=7)
 SLACK_APIKEY = env('SLACK_APIKEY', default=None)
+SLACK_API_USERNAME = env('SLACK_API_USERNAME', default=None)
 
 REST_FRAMEWORK = {
     'DEFAULT_AUTHENTICATION_CLASSES': [
diff --git a/project/slacksync/__init__.py b/project/slacksync/__init__.py
index e69de29b..40a96afc 100644
--- a/project/slacksync/__init__.py
+++ b/project/slacksync/__init__.py
@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-
diff --git a/project/slacksync/admin.py b/project/slacksync/admin.py
index 8c38f3f3..34eec6ab 100644
--- a/project/slacksync/admin.py
+++ b/project/slacksync/admin.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from django.contrib import admin
 
 # Register your models here.
diff --git a/project/slacksync/membersync.py b/project/slacksync/membersync.py
new file mode 100644
index 00000000..80eaa30d
--- /dev/null
+++ b/project/slacksync/membersync.py
@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+from django.conf import settings
+from members.objects import Member
+from requests.sessions import Session
+
+from .utils import api_configured, get_client
+
+logger = logger.getLogger()
+
+
+class SlackMemberSync(object):
+    def get_slack_users_simple(self, slack, exclude_api_user=True):
+        response = slack.users.list()
+        emails = []
+        for member in response.body['members']:
+            if 'email' not in memeber['profile']:
+                # bot or similar
+                continue
+            if exclude_api_user and member['name'] == settings.SLACK_API_USERNAME:
+                continue
+            emails.append((member['id'], member['name'], member['profile']['email']))
+        return emails
+
+    def sync_members(self, autodeactivate=False):
+        """Sync members, NOTE: https://github.com/ErikKalkoken/slackApiDoc/blob/master/users.admin.setInactive.md says 
+        deactivation via API works only on paid tiers"""
+        with Session() as session:
+            slack = get_client(session=session)
+            slack_users = self.get_slack_users_simple(slack)
+            slac_emails = [x[2] for x in slack_users]
+            add_members = Member.objects.exclude(email__in=slack_emails)
+            for member in add_members:
+                try:
+                    resp = slack.users.admin.invite(member.email)
+                    if 'ok' not in resp.body or not resp.body['ok']:
+                        self.logger.error("Could not invite {}, response: {}".format(member.email, response.body))
+                except Exception as e:
+                    logger.exception("Got exception when trying to invite {}".format(member.email))
+            # TODO: check which members should be removed
diff --git a/project/slacksync/models.py b/project/slacksync/models.py
index 71a83623..a978a7cd 100644
--- a/project/slacksync/models.py
+++ b/project/slacksync/models.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from django.db import models
 
 # Create your models here.
diff --git a/project/slacksync/tests.py b/project/slacksync/tests.py
index 7ce503c2..3c380430 100644
--- a/project/slacksync/tests.py
+++ b/project/slacksync/tests.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from django.test import TestCase
 
 # Create your tests here.
diff --git a/project/slacksync/utils.py b/project/slacksync/utils.py
index e31203db..9cd4c42a 100644
--- a/project/slacksync/utils.py
+++ b/project/slacksync/utils.py
@@ -1,9 +1,11 @@
+# -*- coding: utf-8 -*-
 from django.conf import settings
 from slacker import Slacker
 
 
 def api_configured():
-    return bool(settings.SLACK_APIKEY)
+    return bool(settings.SLACK_APIKEY) and bool(settings.SLACK_API_USERNAME)
+
 
 def get_client(**kwargs):
     if not api_configured():
diff --git a/project/slacksync/views.py b/project/slacksync/views.py
index 91ea44a2..d3ff201c 100644
--- a/project/slacksync/views.py
+++ b/project/slacksync/views.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from django.shortcuts import render
 
 # Create your views here.

From bcad20c47a10451c49873805ddb398e8a322c183 Mon Sep 17 00:00:00 2001
From: Eero af Heurlin <eero.afheurlin@iki.fi>
Date: Tue, 30 May 2017 01:08:58 +0300
Subject: [PATCH 06/17] Run code-quality tools

---
 .../management/commands/update_membershipfees.py       |  4 ++--
 project/creditor/tests/fixtures/recurring.py           |  2 ++
 project/examples/handlers.py                           |  4 ++--
 project/ndaparser/models.py                            | 10 +++-------
 project/ndaparser/views.py                             | 10 +++++-----
 5 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/project/creditor/management/commands/update_membershipfees.py b/project/creditor/management/commands/update_membershipfees.py
index 10cf50ed..bc0db0af 100644
--- a/project/creditor/management/commands/update_membershipfees.py
+++ b/project/creditor/management/commands/update_membershipfees.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 import datetime
-import dateutil.parser
 
+import dateutil.parser
 from creditor.models import RecurringTransaction, TransactionTag
 from creditor.tests.fixtures.recurring import MembershipfeeFactory
 from django.core.management.base import BaseCommand, CommandError
@@ -26,7 +26,7 @@ def handle(self, *args, **options):
             end=None,
             start__lt=cutoff_dt,
             amount=options['oldamount']
-            ):
+        ):
             rt.end = end_dt
             rt.save()
             newrt = MembershipfeeFactory.create(amount=options['newamount'], start=cutoff_dt, end=None, owner=rt.owner)
diff --git a/project/creditor/tests/fixtures/recurring.py b/project/creditor/tests/fixtures/recurring.py
index a00ef66c..48728d53 100644
--- a/project/creditor/tests/fixtures/recurring.py
+++ b/project/creditor/tests/fixtures/recurring.py
@@ -9,11 +9,13 @@
 
 from .tags import TransactionTagFactory
 
+
 def get_tag():
     if TransactionTag.objects.count():
         return factory.fuzzy.FuzzyChoice(TransactionTag.objects.all())
     return factory.SubFactory(TransactionTagFactory, label='Membership fee', tmatch='1')
 
+
 class RecurringTransactionFactory(factory.django.DjangoModelFactory):
 
     class Meta:
diff --git a/project/examples/handlers.py b/project/examples/handlers.py
index a3252aa3..93bf2e94 100644
--- a/project/examples/handlers.py
+++ b/project/examples/handlers.py
@@ -166,9 +166,9 @@ def import_generic_transaction(self, at, lt):
         return lt
 
     def import_tmatch_transaction(self, at, lt):
-        if len(at.reference) < 2: # To avoid indexerrors
+        if len(at.reference) < 2:  # To avoid indexerrors
             return None
-        if at.reference[0:2] == "RF": # ISO references, our lookup won't work with them, even worse: there will be exceptions
+        if at.reference[0:2] == "RF":  # ISO references, our lookup won't work with them, even worse: there will be exceptions
             return None
         # In  this example the last meaningful number (last number is checksum) of the reference is used to recognize the TransactionTag
         try:
diff --git a/project/ndaparser/models.py b/project/ndaparser/models.py
index 53cdbc37..f829a242 100644
--- a/project/ndaparser/models.py
+++ b/project/ndaparser/models.py
@@ -1,23 +1,20 @@
 # -*- coding: utf-8 -*-
 import datetime
-import slugify as unicodeslugify
 
-from django.db import models, transaction
+import slugify as unicodeslugify
 from django.conf import settings
 from django.contrib.auth import get_user_model
+from django.db import models, transaction
 from django.utils.translation import ugettext_lazy as _
 
-
 from asylum.models import AsylumModel
 
 
-
 def get_sentinel_user():
     """Gets a "sentinel" user ("deleted") and for assigning as uploader"""
     return get_user_model().objects.get_or_create(username='deleted')[0]
 
 
-
 def datestamped_and_normalized(instance, filename):
     """Normalized filename and places in datestamped path"""
     file_parts = filename.split('.')
@@ -34,7 +31,6 @@ def datestamped_and_normalized(instance, filename):
     return datetime.datetime.now().strftime("ndaparser/%Y/%m/%d/{}").format(filename_normalized)
 
 
-
 class UploadedTransaction(AsylumModel):
     """Track uploaded transaction files"""
     user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.SET(get_sentinel_user))
@@ -45,4 +41,4 @@ class UploadedTransaction(AsylumModel):
     class Meta:
         verbose_name = _('Uploaded transaction')
         verbose_name_plural = _('Uploaded transaction')
-        ordering = [ '-stamp' ]
+        ordering = ['-stamp']
diff --git a/project/ndaparser/views.py b/project/ndaparser/views.py
index 665cce7e..5cec0762 100644
--- a/project/ndaparser/views.py
+++ b/project/ndaparser/views.py
@@ -7,8 +7,8 @@
 
 from .forms import UploadForm
 from .importer import NDAImporter
-from .parser import parseLine
 from .models import UploadedTransaction
+from .parser import parseLine
 
 
 class NordeaUploadView(FormView):
@@ -38,7 +38,7 @@ def form_valid(self, form):
         last_stamp = None
         with open(tmp.name) as fp:
             for line in fp:
-                nt =  parseLine(line)
+                nt = parseLine(line)
                 if not nt:
                     continue
                 if not last_stamp:
@@ -47,9 +47,9 @@ def form_valid(self, form):
                     last_stamp = nt.timestamp
 
         UploadedTransaction(
-            last_transaction = last_stamp,
-            file = self.request.FILES['ndafile'],
-            user = self.request.user
+            last_transaction=last_stamp,
+            file=self.request.FILES['ndafile'],
+            user=self.request.user
         ).save()
 
         # Done with the temp file, get rid of it

From ef7e8f0683dbabb53e715222d3d9bc8681a01661 Mon Sep 17 00:00:00 2001
From: Eero af Heurlin <eero.afheurlin@iki.fi>
Date: Tue, 30 May 2017 21:15:55 +0300
Subject: [PATCH 07/17] Implement the sync

---
 .../management/commands/sync_slack_users.py   | 18 +++++++-
 project/slacksync/membersync.py               | 41 +++++++++++++++++--
 project/slacksync/utils.py                    | 22 ++++++++++
 3 files changed, 76 insertions(+), 5 deletions(-)

diff --git a/project/slacksync/management/commands/sync_slack_users.py b/project/slacksync/management/commands/sync_slack_users.py
index 5ddf1d01..afc54f25 100644
--- a/project/slacksync/management/commands/sync_slack_users.py
+++ b/project/slacksync/management/commands/sync_slack_users.py
@@ -1,12 +1,28 @@
 # -*- coding: utf-8 -*-
 from django.core.management.base import BaseCommand, CommandError
+from slacksync.membersync import SlackMemberSync
+from slacksync.utils import api_configured
 
 
 class Command(BaseCommand):
     help = 'Make sure all members are in Slack and optionally kick non-members'
 
     def add_arguments(self, parser):
+        parser.add_argument('--autodeactivate', action='store_true', help='Automatically deactivate users that are no longer members')
+
         pass
 
     def handle(self, *args, **options):
-        raise NotImplemented()
+        if not api_configured():
+            raise CommandError("API not configured")
+        autoremove = False
+        if options['autodeactivate']:
+            autoremove = True
+        sync = SlackMemberSync(autoremove)
+        tbd = sync.sync_members()
+        if options['verbosity'] > 1:
+            for dm in tbd:
+                if autoremove:
+                    print("User {uid} ({email}) was removed".format(uid=dm[0], email=dm[1]))
+                else:
+                    print("User {uid} ({email}) should be removed".format(uid=dm[0], email=dm[1]))
diff --git a/project/slacksync/membersync.py b/project/slacksync/membersync.py
index 80eaa30d..5447acd0 100644
--- a/project/slacksync/membersync.py
+++ b/project/slacksync/membersync.py
@@ -1,15 +1,21 @@
 # -*- coding: utf-8 -*-
+import logging
+import time
+
 from django.conf import settings
-from members.objects import Member
+from members.models import Member
 from requests.sessions import Session
 
 from .utils import api_configured, get_client
 
-logger = logger.getLogger()
+logger = logging.getLogger()
 
 
 class SlackMemberSync(object):
+    """Sync members and slack members"""
+
     def get_slack_users_simple(self, slack, exclude_api_user=True):
+        """Get just the properties we need from the slack members list"""
         response = slack.users.list()
         emails = []
         for member in response.body['members']:
@@ -24,16 +30,43 @@ def get_slack_users_simple(self, slack, exclude_api_user=True):
     def sync_members(self, autodeactivate=False):
         """Sync members, NOTE: https://github.com/ErikKalkoken/slackApiDoc/blob/master/users.admin.setInactive.md says 
         deactivation via API works only on paid tiers"""
+        if not api_configured():
+            raise RuntimeError("Slack API not configured")
         with Session() as session:
             slack = get_client(session=session)
             slack_users = self.get_slack_users_simple(slack)
-            slac_emails = [x[2] for x in slack_users]
+            slack_emails = set([x[2] for x in slack_users])
             add_members = Member.objects.exclude(email__in=slack_emails)
             for member in add_members:
                 try:
                     resp = slack.users.admin.invite(member.email)
                     if 'ok' not in resp.body or not resp.body['ok']:
                         self.logger.error("Could not invite {}, response: {}".format(member.email, response.body))
+                    time.sleep(0.1)  # rate-limit
                 except Exception as e:
                     logger.exception("Got exception when trying to invite {}".format(member.email))
-            # TODO: check which members should be removed
+
+            member_emails = set(Member.objects.values_list('email', flat=True))
+            remove_slack_emails = slack_emails - member_emails
+            remove_usernames = []
+            if not remove_slack_emails:
+                return remove_usernames
+
+            usernames_by_email = {x[2]: x[1] for x in slack_users}
+            remove_usernames = [(usernames_by_email[x], x) for x in remove_slack_emails]
+
+            if not autodeactivate:
+                return remove_usernames
+
+            userids_by_email = {x[2]: x[0] for x in slack_users}
+            for email in remove_slack_emails:
+                try:
+                    resp = slack.users.admin.setInactive(userids_by_email[email])
+                    if 'ok' not in resp.body or not resp.body['ok']:
+                        self.logger.error(
+                            "Could not deactivate {}, response: {}".format(email, response.body))
+                    time.sleep(0.1)  # rate-limit
+                except Exception as e:
+                    logger.exception("Got exception when trying to deactivate {}".format(email))
+
+            return remove_usernames
diff --git a/project/slacksync/utils.py b/project/slacksync/utils.py
index 9cd4c42a..2196c86a 100644
--- a/project/slacksync/utils.py
+++ b/project/slacksync/utils.py
@@ -1,13 +1,35 @@
 # -*- coding: utf-8 -*-
+import logging
+
 from django.conf import settings
 from slacker import Slacker
 
+logger = logging.getLogger()
+
 
 def api_configured():
+    """Check that Slack API settings are configured"""
     return bool(settings.SLACK_APIKEY) and bool(settings.SLACK_API_USERNAME)
 
 
 def get_client(**kwargs):
+    """Get a Slacker instance"""
     if not api_configured():
         return False
     return Slacker(settings.SLACK_APIKEY, **kwargs)
+
+
+def quick_invite(email):
+    """Quickly invite single email"""
+    if not api_configured():
+        return False
+    slack = get_client()
+    try:
+        resp = slack.users.admin.invite(member.email)
+        if 'ok' not in resp.body or not resp.body['ok']:
+            self.logger.error("Could not invite {}, response: {}".format(email, response.body))
+            return False
+    except Exception as e:
+        logger.exception("Got exception when trying to invite {}".format(email))
+        return False
+    return True

From 15e051abe5c82ebe9f3db3920c425dffd4c139a1 Mon Sep 17 00:00:00 2001
From: Eero af Heurlin <eero.afheurlin@iki.fi>
Date: Thu, 1 Jun 2017 19:15:07 +0300
Subject: [PATCH 08/17] Fix autoremove in wrong place

---
 project/slacksync/management/commands/sync_slack_users.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/project/slacksync/management/commands/sync_slack_users.py b/project/slacksync/management/commands/sync_slack_users.py
index afc54f25..621d3049 100644
--- a/project/slacksync/management/commands/sync_slack_users.py
+++ b/project/slacksync/management/commands/sync_slack_users.py
@@ -18,8 +18,8 @@ def handle(self, *args, **options):
         autoremove = False
         if options['autodeactivate']:
             autoremove = True
-        sync = SlackMemberSync(autoremove)
-        tbd = sync.sync_members()
+        sync = SlackMemberSync()
+        tbd = sync.sync_members(autoremove)
         if options['verbosity'] > 1:
             for dm in tbd:
                 if autoremove:

From d3178b53badd0689ed9c813a31827f7e82ee513c Mon Sep 17 00:00:00 2001
From: Eero af Heurlin <eero.afheurlin@iki.fi>
Date: Thu, 1 Jun 2017 19:17:56 +0300
Subject: [PATCH 09/17] fix typos, variable names

---
 project/slacksync/membersync.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/project/slacksync/membersync.py b/project/slacksync/membersync.py
index 5447acd0..395a8369 100644
--- a/project/slacksync/membersync.py
+++ b/project/slacksync/membersync.py
@@ -19,7 +19,7 @@ def get_slack_users_simple(self, slack, exclude_api_user=True):
         response = slack.users.list()
         emails = []
         for member in response.body['members']:
-            if 'email' not in memeber['profile']:
+            if 'email' not in member['profile']:
                 # bot or similar
                 continue
             if exclude_api_user and member['name'] == settings.SLACK_API_USERNAME:
@@ -41,7 +41,7 @@ def sync_members(self, autodeactivate=False):
                 try:
                     resp = slack.users.admin.invite(member.email)
                     if 'ok' not in resp.body or not resp.body['ok']:
-                        self.logger.error("Could not invite {}, response: {}".format(member.email, response.body))
+                        self.logger.error("Could not invite {}, response: {}".format(member.email, resp.body))
                     time.sleep(0.1)  # rate-limit
                 except Exception as e:
                     logger.exception("Got exception when trying to invite {}".format(member.email))
@@ -64,7 +64,7 @@ def sync_members(self, autodeactivate=False):
                     resp = slack.users.admin.setInactive(userids_by_email[email])
                     if 'ok' not in resp.body or not resp.body['ok']:
                         self.logger.error(
-                            "Could not deactivate {}, response: {}".format(email, response.body))
+                            "Could not deactivate {}, response: {}".format(email, resp.body))
                     time.sleep(0.1)  # rate-limit
                 except Exception as e:
                     logger.exception("Got exception when trying to deactivate {}".format(email))

From e70ea92185519eaa49e54828468e1588fbbb9945 Mon Sep 17 00:00:00 2001
From: Eero af Heurlin <eero.afheurlin@iki.fi>
Date: Thu, 1 Jun 2017 19:38:40 +0300
Subject: [PATCH 10/17] handle rate-limit exceptions

---
 .../management/commands/sync_slack_users.py   |  6 ++-
 project/slacksync/membersync.py               | 42 ++++++++++++++-----
 2 files changed, 36 insertions(+), 12 deletions(-)

diff --git a/project/slacksync/management/commands/sync_slack_users.py b/project/slacksync/management/commands/sync_slack_users.py
index 621d3049..fdece52e 100644
--- a/project/slacksync/management/commands/sync_slack_users.py
+++ b/project/slacksync/management/commands/sync_slack_users.py
@@ -9,6 +9,7 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument('--autodeactivate', action='store_true', help='Automatically deactivate users that are no longer members')
+        parser.add_argument('--noresend', action='store_true', help='Automatically deactivate users that are no longer members')
 
         pass
 
@@ -18,8 +19,11 @@ def handle(self, *args, **options):
         autoremove = False
         if options['autodeactivate']:
             autoremove = True
+        resend = True
+        if options['noresend']:
+            resend = False
         sync = SlackMemberSync()
-        tbd = sync.sync_members(autoremove)
+        tbd = sync.sync_members(autoremove, )
         if options['verbosity'] > 1:
             for dm in tbd:
                 if autoremove:
diff --git a/project/slacksync/membersync.py b/project/slacksync/membersync.py
index 395a8369..8aa0bea8 100644
--- a/project/slacksync/membersync.py
+++ b/project/slacksync/membersync.py
@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+import collections
 import logging
 import time
 
@@ -27,7 +28,7 @@ def get_slack_users_simple(self, slack, exclude_api_user=True):
             emails.append((member['id'], member['name'], member['profile']['email']))
         return emails
 
-    def sync_members(self, autodeactivate=False):
+    def sync_members(self, autodeactivate=False, resend=True):
         """Sync members, NOTE: https://github.com/ErikKalkoken/slackApiDoc/blob/master/users.admin.setInactive.md says 
         deactivation via API works only on paid tiers"""
         if not api_configured():
@@ -36,15 +37,25 @@ def sync_members(self, autodeactivate=False):
             slack = get_client(session=session)
             slack_users = self.get_slack_users_simple(slack)
             slack_emails = set([x[2] for x in slack_users])
-            add_members = Member.objects.exclude(email__in=slack_emails)
-            for member in add_members:
+            add_members = collections.deque(Member.objects.exclude(email__in=slack_emails))
+
+            while add_members:
+                member = add_members.popleft()
                 try:
-                    resp = slack.users.admin.invite(member.email)
+                    resp = slack.users.admin.invite(member.email, resend=resend)
                     if 'ok' not in resp.body or not resp.body['ok']:
                         self.logger.error("Could not invite {}, response: {}".format(member.email, resp.body))
-                    time.sleep(0.1)  # rate-limit
+                    time.sleep(0.25)  # rate-limit
                 except Exception as e:
-                    logger.exception("Got exception when trying to invite {}".format(member.email))
+                    if 'Retry-After' in e.response.headers:
+                        wait_s = int(e.response.headers['Retry-After'])
+                        logger.warning("Asked to wait {}s".format(wait_s))
+                        time.sleep(wait_s)
+                        add_members.appendleft(member)
+                        continue
+                    else:
+                        logger.exception("Got exception when trying to invite {}".format(member.email))
+                        raise e
 
             member_emails = set(Member.objects.values_list('email', flat=True))
             remove_slack_emails = slack_emails - member_emails
@@ -59,14 +70,23 @@ def sync_members(self, autodeactivate=False):
                 return remove_usernames
 
             userids_by_email = {x[2]: x[0] for x in slack_users}
-            for email in remove_slack_emails:
+            remove_iter = collections.deque(remove_slack_emails)
+            while remove_iter:
+                email = remove_iter.popleft()
                 try:
                     resp = slack.users.admin.setInactive(userids_by_email[email])
                     if 'ok' not in resp.body or not resp.body['ok']:
-                        self.logger.error(
-                            "Could not deactivate {}, response: {}".format(email, resp.body))
-                    time.sleep(0.1)  # rate-limit
+                        self.logger.error("Could not deactivate {}, response: {}".format(email, resp.body))
+                    time.sleep(0.25)  # rate-limit
                 except Exception as e:
-                    logger.exception("Got exception when trying to deactivate {}".format(email))
+                    if 'Retry-After' in e.response.headers:
+                        wait_s = int(e.response.headers['Retry-After'])
+                        logger.warning("Asked to wait {}s".format(wait_s))
+                        time.sleep(wait_s)
+                        remove_iter.appendleft(email)
+                        continue
+                    else:
+                        logger.exception("Got exception when trying to deactivate {}".format(email))
+                        raise e
 
             return remove_usernames

From 9b4dfccae39e6742e54e87bd26e6c8bf1ce18f8e Mon Sep 17 00:00:00 2001
From: Eero af Heurlin <eero.afheurlin@iki.fi>
Date: Thu, 1 Jun 2017 19:44:30 +0300
Subject: [PATCH 11/17] wait a bit longer, log the email we wait for

---
 project/slacksync/membersync.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/project/slacksync/membersync.py b/project/slacksync/membersync.py
index 8aa0bea8..23dafefc 100644
--- a/project/slacksync/membersync.py
+++ b/project/slacksync/membersync.py
@@ -49,8 +49,8 @@ def sync_members(self, autodeactivate=False, resend=True):
                 except Exception as e:
                     if 'Retry-After' in e.response.headers:
                         wait_s = int(e.response.headers['Retry-After'])
-                        logger.warning("Asked to wait {}s".format(wait_s))
-                        time.sleep(wait_s)
+                        logger.warning("Asked to wait {}s before retrying invite for {}".format(wait_s, member.email))
+                        time.sleep(wait_s*1.5)
                         add_members.appendleft(member)
                         continue
                     else:
@@ -81,8 +81,8 @@ def sync_members(self, autodeactivate=False, resend=True):
                 except Exception as e:
                     if 'Retry-After' in e.response.headers:
                         wait_s = int(e.response.headers['Retry-After'])
-                        logger.warning("Asked to wait {}s".format(wait_s))
-                        time.sleep(wait_s)
+                        logger.warning("Asked to wait {}s before retrying deactivation for {}".format(wait_s, email))
+                        time.sleep(wait_s*1.5)
                         remove_iter.appendleft(email)
                         continue
                     else:

From 937861c7603ff396c55e2cc340bf45c9e5932613 Mon Sep 17 00:00:00 2001
From: Eero af Heurlin <eero.afheurlin@iki.fi>
Date: Thu, 1 Jun 2017 19:55:40 +0300
Subject: [PATCH 12/17] Pass the noresend correctly, use more specific
 exceptions

---
 .../slacksync/management/commands/sync_slack_users.py  |  2 +-
 project/slacksync/membersync.py                        | 10 ++++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/project/slacksync/management/commands/sync_slack_users.py b/project/slacksync/management/commands/sync_slack_users.py
index fdece52e..91dc108d 100644
--- a/project/slacksync/management/commands/sync_slack_users.py
+++ b/project/slacksync/management/commands/sync_slack_users.py
@@ -23,7 +23,7 @@ def handle(self, *args, **options):
         if options['noresend']:
             resend = False
         sync = SlackMemberSync()
-        tbd = sync.sync_members(autoremove, )
+        tbd = sync.sync_members(autoremove, resend)
         if options['verbosity'] > 1:
             for dm in tbd:
                 if autoremove:
diff --git a/project/slacksync/membersync.py b/project/slacksync/membersync.py
index 23dafefc..0a045bee 100644
--- a/project/slacksync/membersync.py
+++ b/project/slacksync/membersync.py
@@ -6,6 +6,8 @@
 from django.conf import settings
 from members.models import Member
 from requests.sessions import Session
+from requests.exceptions import RequestException
+import slacker
 
 from .utils import api_configured, get_client
 
@@ -46,7 +48,11 @@ def sync_members(self, autodeactivate=False, resend=True):
                     if 'ok' not in resp.body or not resp.body['ok']:
                         self.logger.error("Could not invite {}, response: {}".format(member.email, resp.body))
                     time.sleep(0.25)  # rate-limit
-                except Exception as e:
+                except slacker.Error as e:
+                    if str(e) == 'sent_recently':
+                        continue
+                    raise e
+                except RequestException as e:
                     if 'Retry-After' in e.response.headers:
                         wait_s = int(e.response.headers['Retry-After'])
                         logger.warning("Asked to wait {}s before retrying invite for {}".format(wait_s, member.email))
@@ -78,7 +84,7 @@ def sync_members(self, autodeactivate=False, resend=True):
                     if 'ok' not in resp.body or not resp.body['ok']:
                         self.logger.error("Could not deactivate {}, response: {}".format(email, resp.body))
                     time.sleep(0.25)  # rate-limit
-                except Exception as e:
+                except RequestException as e:
                     if 'Retry-After' in e.response.headers:
                         wait_s = int(e.response.headers['Retry-After'])
                         logger.warning("Asked to wait {}s before retrying deactivation for {}".format(wait_s, email))

From f8d5fa8d9b98b6b8f8b114b7eaac11777ecb7ad0 Mon Sep 17 00:00:00 2001
From: Eero af Heurlin <eero.afheurlin@iki.fi>
Date: Thu, 1 Jun 2017 20:09:57 +0300
Subject: [PATCH 13/17] append retries to right

---
 project/slacksync/membersync.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/project/slacksync/membersync.py b/project/slacksync/membersync.py
index 0a045bee..1344f112 100644
--- a/project/slacksync/membersync.py
+++ b/project/slacksync/membersync.py
@@ -57,7 +57,7 @@ def sync_members(self, autodeactivate=False, resend=True):
                         wait_s = int(e.response.headers['Retry-After'])
                         logger.warning("Asked to wait {}s before retrying invite for {}".format(wait_s, member.email))
                         time.sleep(wait_s*1.5)
-                        add_members.appendleft(member)
+                        add_members.append(member)
                         continue
                     else:
                         logger.exception("Got exception when trying to invite {}".format(member.email))
@@ -89,7 +89,7 @@ def sync_members(self, autodeactivate=False, resend=True):
                         wait_s = int(e.response.headers['Retry-After'])
                         logger.warning("Asked to wait {}s before retrying deactivation for {}".format(wait_s, email))
                         time.sleep(wait_s*1.5)
-                        remove_iter.appendleft(email)
+                        remove_iter.append(email)
                         continue
                     else:
                         logger.exception("Got exception when trying to deactivate {}".format(email))

From 4c573118f07992cc2623511d8886e418c7127502 Mon Sep 17 00:00:00 2001
From: Eero af Heurlin <eero.afheurlin@iki.fi>
Date: Thu, 1 Jun 2017 20:15:53 +0300
Subject: [PATCH 14/17] Run code-quality -tools

---
 project/slacksync/membersync.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/project/slacksync/membersync.py b/project/slacksync/membersync.py
index 1344f112..fe350c6e 100644
--- a/project/slacksync/membersync.py
+++ b/project/slacksync/membersync.py
@@ -3,11 +3,11 @@
 import logging
 import time
 
+import slacker
 from django.conf import settings
 from members.models import Member
-from requests.sessions import Session
 from requests.exceptions import RequestException
-import slacker
+from requests.sessions import Session
 
 from .utils import api_configured, get_client
 
@@ -51,12 +51,15 @@ def sync_members(self, autodeactivate=False, resend=True):
                 except slacker.Error as e:
                     if str(e) == 'sent_recently':
                         continue
+                    if str(e) == 'invalid_email':
+                        logger.error("Slack says {} is invalid_email".format(member.email))
+                        continue
                     raise e
                 except RequestException as e:
                     if 'Retry-After' in e.response.headers:
                         wait_s = int(e.response.headers['Retry-After'])
                         logger.warning("Asked to wait {}s before retrying invite for {}".format(wait_s, member.email))
-                        time.sleep(wait_s*1.5)
+                        time.sleep(wait_s * 1.5)
                         add_members.append(member)
                         continue
                     else:
@@ -88,7 +91,7 @@ def sync_members(self, autodeactivate=False, resend=True):
                     if 'Retry-After' in e.response.headers:
                         wait_s = int(e.response.headers['Retry-After'])
                         logger.warning("Asked to wait {}s before retrying deactivation for {}".format(wait_s, email))
-                        time.sleep(wait_s*1.5)
+                        time.sleep(wait_s * 1.5)
                         remove_iter.append(email)
                         continue
                     else:

From 3c70011067b57e88cab4a68adbc92f14b75af4a6 Mon Sep 17 00:00:00 2001
From: Eero af Heurlin <eero.afheurlin@iki.fi>
Date: Tue, 25 Jul 2017 20:07:58 +0300
Subject: [PATCH 15/17] Add optional invite-link handling

---
 project/config/settings/common.py |  1 +
 project/slacksync/membersync.py   | 16 +++++++++++++++-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/project/config/settings/common.py b/project/config/settings/common.py
index aef34dbf..675ef47d 100644
--- a/project/config/settings/common.py
+++ b/project/config/settings/common.py
@@ -273,6 +273,7 @@
 HOLVI_NOTIFICATION_INTERVAL_DAYS = env('HOLVI_NOTIFICATION_INTERVAL_DAYS', default=7)
 SLACK_APIKEY = env('SLACK_APIKEY', default=None)
 SLACK_API_USERNAME = env('SLACK_API_USERNAME', default=None)
+SLACK_INVITE_LINK = env('SLACK_INVITE_LINK', default=None)
 
 REST_FRAMEWORK = {
     'DEFAULT_AUTHENTICATION_CLASSES': [
diff --git a/project/slacksync/membersync.py b/project/slacksync/membersync.py
index fe350c6e..65a8dcd4 100644
--- a/project/slacksync/membersync.py
+++ b/project/slacksync/membersync.py
@@ -5,6 +5,7 @@
 
 import slacker
 from django.conf import settings
+from django.core.mail import send_mail
 from members.models import Member
 from requests.exceptions import RequestException
 from requests.sessions import Session
@@ -30,8 +31,17 @@ def get_slack_users_simple(self, slack, exclude_api_user=True):
             emails.append((member['id'], member['name'], member['profile']['email']))
         return emails
 
+    def email_slack_link(self, member):
+        send_mail(
+            "Slack invite to {}".format(settings.ORGANIZATION_NAME),
+            "Click on the link to continue\n\n{}\n".format(settings.SLACK_INVITE_LINK),
+            settings.DEFAULT_FROM_EMAIL,
+            [member.email],
+            fail_silently=True
+        )
+
     def sync_members(self, autodeactivate=False, resend=True):
-        """Sync members, NOTE: https://github.com/ErikKalkoken/slackApiDoc/blob/master/users.admin.setInactive.md says 
+        """Sync members, NOTE: https://github.com/ErikKalkoken/slackApiDoc/blob/master/users.admin.setInactive.md says
         deactivation via API works only on paid tiers"""
         if not api_configured():
             raise RuntimeError("Slack API not configured")
@@ -43,6 +53,10 @@ def sync_members(self, autodeactivate=False, resend=True):
 
             while add_members:
                 member = add_members.popleft()
+                # If we have configured invite-link use it instead of API (which might hit the "too many invites" -error
+                if settings.SLACK_INVITE_LINK:
+                    self.email_slack_link(member)
+                    continue
                 try:
                     resp = slack.users.admin.invite(member.email, resend=resend)
                     if 'ok' not in resp.body or not resp.body['ok']:

From 55caacd182dd29d88de6a809885a018b0aad9363 Mon Sep 17 00:00:00 2001
From: Eero af Heurlin <eero.afheurlin@iki.fi>
Date: Wed, 6 Sep 2017 10:03:26 +0300
Subject: [PATCH 16/17] Fix copy-paste -error

---
 project/slacksync/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/project/slacksync/utils.py b/project/slacksync/utils.py
index 2196c86a..cc4c455a 100644
--- a/project/slacksync/utils.py
+++ b/project/slacksync/utils.py
@@ -25,7 +25,7 @@ def quick_invite(email):
         return False
     slack = get_client()
     try:
-        resp = slack.users.admin.invite(member.email)
+        resp = slack.users.admin.invite(email)
         if 'ok' not in resp.body or not resp.body['ok']:
             self.logger.error("Could not invite {}, response: {}".format(email, response.body))
             return False

From 175ef4cfac9592027c9bf3a8d12e2fcd10422f50 Mon Sep 17 00:00:00 2001
From: Eero af Heurlin <eero.afheurlin@iki.fi>
Date: Thu, 31 Jan 2019 17:53:42 +0200
Subject: [PATCH 17/17] Fix copy-paste error in option text

---
 project/slacksync/management/commands/sync_slack_users.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/project/slacksync/management/commands/sync_slack_users.py b/project/slacksync/management/commands/sync_slack_users.py
index 91dc108d..2450ba9a 100644
--- a/project/slacksync/management/commands/sync_slack_users.py
+++ b/project/slacksync/management/commands/sync_slack_users.py
@@ -9,7 +9,7 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument('--autodeactivate', action='store_true', help='Automatically deactivate users that are no longer members')
-        parser.add_argument('--noresend', action='store_true', help='Automatically deactivate users that are no longer members')
+        parser.add_argument('--noresend', action='store_true', help='Do not resend the invitation mail')
 
         pass