Store media usn per-file

Since anki.media.MediaManager does not store per-file usn (as it's
not really needed for anything in the client), this requires us to
drop it and implement a custom media manager.

ankisyncd/collection.py

@@ -1,6 +1,8 @@
 import anki
 import anki.storage
 
+import ankisyncd.media
+
 import os, errno
 import logging
 
@@ -63,7 +65,13 @@ class CollectionWrapper:
         return col
 
     def _get_collection(self):
-        return anki.storage.Collection(self.path)
+        col = anki.storage.Collection(self.path)
+
+        # Ugly hack, replace default media manager with our custom one
+        col.media.close()
+        col.media = ankisyncd.media.ServerMediaManager(col)
+
+        return col
 
     def open(self):
         """Open the collection, or create it if it doesn't exist."""

ankisyncd/media.py

@@ -0,0 +1,56 @@
+# Based on anki.media.MediaManager, © Ankitects Pty Ltd and contributors
+# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+# Original source: https://raw.githubusercontent.com/dae/anki/62481ddc1aa78430cb8114cbf00a7739824318a8/anki/media.py
+
+import re
+import os
+import os.path
+
+import anki.db
+
+
+class ServerMediaManager:
+    def __init__(self, col):
+        self._dir = re.sub(r"(?i)\.(anki2)$", ".media", col.path)
+        self.connect()
+
+    def connect(self):
+        path = self.dir() + ".server.db"
+        # TODO: migrate old db
+        create = not os.path.exists(path)
+        self.db = anki.db.DB(path)
+        if create:
+            self.db.executescript(
+                """CREATE TABLE media (
+                       fname TEXT NOT NULL PRIMARY KEY,
+                       usn INT NOT NULL,
+                       csum TEXT -- null if deleted
+                   );
+                CREATE INDEX idx_media_usn ON media (usn);"""
+            )
+
+    def close(self):
+        self.db.close()
+
+    def dir(self):
+        return self._dir
+
+    def lastUsn(self):
+        return self.db.scalar("SELECT usn FROM media ORDER BY usn DESC LIMIT 1") or 0
+
+    def mediaCount(self):
+        return self.db.scalar("SELECT count() FROM media WHERE csum IS NOT NULL")
+
+    # used only in unit tests
+    def syncInfo(self, fname):
+        return self.db.first("SELECT csum, 0 FROM media WHERE fname=?", fname)
+
+    def syncDelete(self, fname):
+        fpath = os.path.join(self.dir(), fname)
+        if os.path.exists(fpath):
+            os.remove(fpath)
+        self.db.execute(
+            "UPDATE media SET csum = NULL, usn = ? WHERE fname = ?",
+            self.lastUsn() + 1,
+            fname,
+        )

ankisyncd/sync_app.py

@@ -175,7 +175,7 @@ class SyncMediaHandler:
     operations = ['begin', 'mediaChanges', 'mediaSanity', 'uploadChanges', 'downloadFiles']
 
     def __init__(self, col):
-        anki.sync.MediaSyncer.__init__(self, col)
+        self.col = col
 
     def begin(self, skey):
         return {
@@ -196,11 +196,6 @@ class SyncMediaHandler:
             self._check_zip_data(z)
             processed_count = self._adopt_media_changes_from_zip(z)
 
-        # We increment our lastUsn once for each file we processed.
-        # (lastUsn - processed_count) must equal the client's lastUsn.
-        our_last_usn = self.col.media.lastUsn()
-        self.col.media.setLastUsn(our_last_usn + processed_count)
-
         return {
             'data': [processed_count, self.col.media.lastUsn()],
             'err': '',
@@ -238,6 +233,8 @@ class SyncMediaHandler:
 
         # Add media files that were added on the client.
         media_to_add = []
+        usn = self.col.media.lastUsn()
+        oldUsn = usn
         for i in zip_file.infolist():
             if i.filename == "_meta":  # Ignore previously retrieved metadata.
                 continue
@@ -250,9 +247,9 @@ class SyncMediaHandler:
             # Save file to media directory.
             with open(file_path, 'wb') as f:
                 f.write(file_data)
-            mtime = self.col.media._mtime(file_path)
 
-            media_to_add.append((filename, csum, mtime, 0))
+            usn += 1
+            media_to_add.append((filename, usn, csum))
 
         # We count all files we are to remove, even if we don't have them in
         # our media directory and our db doesn't know about them.
@@ -265,8 +262,10 @@ class SyncMediaHandler:
 
         if media_to_add:
             self.col.media.db.executemany(
-                "INSERT OR REPLACE INTO media VALUES (?,?,?,?)", media_to_add)
+                "INSERT OR REPLACE INTO media VALUES (?,?,?)", media_to_add)
+            self.col.media.db.commit()
 
+        assert self.col.media.lastUsn() == oldUsn + processed_count  # TODO: move to some unit test
         return processed_count
 
     @staticmethod
@@ -289,18 +288,11 @@ class SyncMediaHandler:
         Marks all files in list filenames as deleted and removes them from the
         media directory.
         """
-
-        # Mark the files as deleted in our db.
-        self.col.media.db.executemany("UPDATE media " +
-                                      "SET csum = NULL " +
-                                      " WHERE fname = ?",
-                                      [(f, ) for f in filenames])
-
-        # Remove the files from our media directory if it is present.
         logger.debug('Removing %d files from media dir.' % len(filenames))
         for filename in filenames:
             try:
-                os.remove(os.path.join(self.col.media.dir(), filename))
+                self.col.media.syncDelete(filename)
+                self.col.media.db.commit()
             except OSError as err:
                 logger.error("Error when removing file '%s' from media dir: "
                               "%s" % (filename, str(err)))
@@ -330,7 +322,7 @@ class SyncMediaHandler:
         fname = csum = None
 
         if lastUsn < usn or lastUsn == 0:
-            for fname,mtime,csum, in self.col.media.db.execute("select fname,mtime,csum from media"):
+            for fname,usn,csum, in self.col.media.db.execute("select fname,usn,csum from media"):
                 result.append([fname, usn, csum])
 
         return {'data': result, 'err': ''}