implement reserved users for mau limits

synapse/app/homeserver.py

@@ -518,6 +518,8 @@ def run(hs):
     # If you increase the loop period, the accuracy of user_daily_visits
     # table will decrease
     clock.looping_call(generate_user_daily_visit_stats, 5 * 60 * 1000)
+
+    # monthly active user limiting functionality
     clock.looping_call(
         hs.get_datastore().reap_monthly_active_users, 1000 * 60 * 60
     )
@@ -530,9 +532,13 @@ def run(hs):
         current_mau_gauge.set(float(count))
         max_mau_value_gauge.set(float(hs.config.max_mau_value))
 
+    hs.get_datastore().initialise_reserved_users(
+        hs.config.mau_limits_reserved_threepids
+    )
     generate_monthly_active_users()
     if hs.config.limit_usage_by_mau:
         clock.looping_call(generate_monthly_active_users, 5 * 60 * 1000)
+    # End of monthly active user settings
 
     if hs.config.report_stats:
         logger.info("Scheduling stats reporting for 3 hour intervals")

synapse/config/server.py

@@ -75,7 +75,7 @@ class ServerConfig(Config):
                 "max_mau_value", 0,
             )
         self.mau_limits_reserved_threepids = config.get(
-            "mau_limit_reserved_threepid", []
+            "mau_limit_reserved_threepids", []
         )
 
         # FIXME: federation_domain_whitelist needs sytests

synapse/storage/monthly_active_users.py

@@ -12,6 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import logging
 
 from twisted.internet import defer
 
@@ -19,6 +20,7 @@ from synapse.util.caches.descriptors import cached
 
 from ._base import SQLBaseStore
 
+logger = logging.getLogger(__name__)
 
 # Number of msec of granularity to store the monthly_active_user timestamp
 # This means it is not necessary to update the table on every request
@@ -26,24 +28,31 @@ LAST_SEEN_GRANULARITY = 60 * 60 * 1000
 
 
 class MonthlyActiveUsersStore(SQLBaseStore):
-    @defer.inlineCallbacks
     def __init__(self, dbconn, hs):
         super(MonthlyActiveUsersStore, self).__init__(None, hs)
         self._clock = hs.get_clock()
         self.hs = hs
-        threepids = self.hs.config.mau_limits_reserved_threepids
-        self.reserved_user_ids = set()
+        self.reserved_users = ()
+
+    @defer.inlineCallbacks
+    def initialise_reserved_users(self, threepids):
+        # TODO Why can't I do this in init?
+        store = self.hs.get_datastore()
+        reserved_user_list = []
         for tp in threepids:
-            user_id = yield hs.get_datastore().get_user_id_by_threepid(
+            user_id = yield store.get_user_id_by_threepid(
                 tp["medium"], tp["address"]
             )
             if user_id:
-                self.reserved_user_ids.add(user_id)
+                self.upsert_monthly_active_user(user_id)
+                reserved_user_list.append(user_id)
             else:
                 logger.warning(
                     "mau limit reserved threepid %s not found in db" % tp
                 )
+        self.reserved_users = tuple(reserved_user_list)
 
+    @defer.inlineCallbacks
     def reap_monthly_active_users(self):
         """
         Cleans out monthly active user table to ensure that no stale
@@ -58,8 +67,20 @@ class MonthlyActiveUsersStore(SQLBaseStore):
                 int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)
             )
             # Purge stale users
-            sql = "DELETE FROM monthly_active_users WHERE timestamp < ?"
-            txn.execute(sql, (thirty_days_ago,))
+
+            # questionmarks is a hack to overcome sqlite not supporting
+            # tuples in 'WHERE IN %s'
+            questionmarks = '?' * len(self.reserved_users)
+            query_args = [thirty_days_ago]
+            query_args.extend(self.reserved_users)
+
+            sql = """
+                DELETE FROM monthly_active_users
+                WHERE timestamp < ?
+                AND user_id NOT IN ({})
+                """.format(','.join(questionmarks))
+
+            txn.execute(sql, query_args)
 
             # If MAU user count still exceeds the MAU threshold, then delete on
             # a least recently active basis.
@@ -69,6 +90,8 @@ class MonthlyActiveUsersStore(SQLBaseStore):
             # While Postgres does not require 'LIMIT', but also does not support
             # negative LIMIT values. So there is no way to write it that both can
             # support
+            query_args = [self.hs.config.max_mau_value]
+            query_args.extend(self.reserved_users)
             sql = """
                 DELETE FROM monthly_active_users
                 WHERE user_id NOT IN (
@@ -76,8 +99,9 @@ class MonthlyActiveUsersStore(SQLBaseStore):
                     ORDER BY timestamp DESC
                     LIMIT ?
                     )
-                """
-            txn.execute(sql, (self.hs.config.max_mau_value,))
+                AND user_id NOT IN ({})
+                """.format(','.join(questionmarks))
+            txn.execute(sql, query_args)
 
         yield self.runInteraction("reap_monthly_active_users", _reap_users)
         # It seems poor to invalidate the whole cache, Postgres supports
@@ -136,7 +160,7 @@ class MonthlyActiveUsersStore(SQLBaseStore):
             Arguments:
                 user_id (str): user to add/update
             Return:
-                int : timestamp since last seen, None if never seen
+                Deferred[int] : timestamp since last seen, None if never seen
 
         """
 
@@ -158,7 +182,6 @@ class MonthlyActiveUsersStore(SQLBaseStore):
         Args:
             user_id(str): the user_id to query
         """
-
         if self.hs.config.limit_usage_by_mau:
             last_seen_timestamp = yield self._user_last_seen_monthly_active(user_id)
             now = self.hs.get_clock().time_msec()