storage: implemented filesystem

test coverage to be improved

config.yml.example

@@ -3,3 +3,5 @@
 SECRET_KEY: ""
 
 # BCRYPT_LOG_ROUNDS: 13
+# SACHET_STORAGE: "filesystem"
+# SACHET_FILE_DIR: "/srv/sachet/storage"

sachet/server/__init__.py

@@ -21,6 +21,17 @@ bcrypt = Bcrypt(app)
 db = SQLAlchemy(app)
 ma = Marshmallow()
 
+_storage_method = app.config["SACHET_STORAGE"]
+
+storage = None
+
+from sachet.storage import FileSystem
+
+if _storage_method == "filesystem":
+    storage = FileSystem()
+else:
+    raise ValueError(f"{_storage_method} is not a valid storage method.")
+
 import sachet.server.commands
 
 from sachet.server.users.views import users_blueprint

sachet/server/config.py

@@ -8,16 +8,20 @@ class BaseConfig:
     SQLALCHEMY_DATABASE_URI = sqlalchemy_base + ".db"
     BCRYPT_LOG_ROUNDS = 13
     SQLALCHEMY_TRACK_MODIFICATIONS = False
+    SACHET_STORAGE = "filesystem"
+    SACHET_FILE_DIR = "/srv/sachet/storage"
 
 
 class TestingConfig(BaseConfig):
     SQLALCHEMY_DATABASE_URI = sqlalchemy_base + "_test" + ".db"
     BCRYPT_LOG_ROUNDS = 4
+    SACHET_FILE_DIR = "storage_test"
 
 
 class DevelopmentConfig(BaseConfig):
     SQLALCHEMY_DATABASE_URI = sqlalchemy_base + "_dev" + ".db"
     BCRYPT_LOG_ROUNDS = 4
+    SACHET_FILE_DIR = "storage_dev"
 
 
 class ProductionConfig(BaseConfig):

sachet/storage/__init__.py

@@ -0,0 +1,71 @@
+class Storage:
+    """Generic storage interface.
+
+    Raises:
+        OSError if SACHET_FILE_DIR could not be opened as a directory.
+    """
+
+    def create(self, name):
+        """Create file along with the metadata.
+
+        Raises:
+            OSError if the file already exists.
+        """
+
+        pass
+
+    def open(self, name, mode="r"):
+        """Open a file for reading/writing.
+
+        Raises:
+            OSError if the file does not exist.
+
+        Returns:
+            Stream to access file (similar to open()'s handle.)
+        """
+
+        pass
+
+    def read_metadata(self, name):
+        """Get metadata for a file.
+
+        Raises:
+            OSError if the file does not exist.
+        """
+
+        pass
+
+    def write_metadata(self, name, data):
+        """Set metadata for a file.
+
+        Raises:
+            OSError if the file does not exist.
+        """
+
+        pass
+
+    def delete(self, name):
+        """Delete file and associated metadata.
+
+        Raises:
+            OSError if the file does not exist.
+        """
+
+        pass
+
+    def rename(self, name, new_name):
+        """Rename a file.
+
+        Raises:
+            OSError if the file does not exist.
+        """
+
+        pass
+
+    def list_files(self):
+        """Lists all files."""
+
+        pass
+
+
+from .filesystem import FileSystem

sachet/storage/filesystem.py

@@ -0,0 +1,95 @@
+from sachet.storage import Storage
+from pathlib import Path
+from sachet.server import app
+from werkzeug.utils import secure_filename
+import json
+
+
+class FileSystem(Storage):
+    def __init__(self):
+        config_path = Path(app.config["SACHET_FILE_DIR"])
+        if config_path.is_absolute():
+            self._directory = config_path
+        else:
+            self._directory = Path(app.instance_path) / config_path
+
+        self._files_directory = self._directory / Path("files")
+        self._meta_directory = self._directory / Path("meta")
+
+        self._files_directory.mkdir(mode=0o700, exist_ok=True, parents=True)
+        self._meta_directory.mkdir(mode=0o700, exist_ok=True, parents=True)
+
+        if not self._directory.is_dir():
+            raise OSError(f"'{app.config['SACHET_FILE_DIR']}' is not a directory.")
+
+    def _get_path(self, name):
+        name = secure_filename(name)
+        return self._files_directory / Path(name)
+
+    def _get_meta_path(self, name):
+        name = secure_filename(name)
+        return self._meta_directory / Path(name)
+
+    def create(self, name):
+        path = self._get_path(name)
+        if path.exists():
+            raise OSError(f"Path {path} already exists.")
+
+        meta_path = self._get_meta_path(name)
+        if meta_path.exists():
+            raise OSError(f"Path {meta_path} already exists.")
+
+        path.touch()
+        meta_path.touch()
+
+    def delete(self, name):
+        path = self._get_path(name)
+        if not path.exists():
+            raise OSError(f"Path {path} does not exist.")
+        path.unlink()
+
+    def open(self, name, mode="r"):
+        path = self._get_path(name)
+        if not path.exists():
+            raise OSError(f"Path {path} does not exist.")
+
+        return path.open(mode=mode)
+
+    def read_metadata(self, name):
+        meta_path = self._get_meta_path(name)
+        if not meta_path.exists():
+            raise OSError(f"Path {meta_path} does not exist.")
+
+        with meta_path.open() as meta_file:
+            content = meta_file.read()
+            return json.loads(content)
+
+    def write_metadata(self, name, data):
+        meta_path = self._get_meta_path(name)
+        if not meta_path.exists():
+            raise OSError(f"Path {meta_path} does not exist.")
+
+        with meta_path.open("w") as meta_file:
+            content = json.dumps(data)
+            meta_file.write(content)
+
+    def rename(self, name, new_name):
+        path = self._get_path(name)
+        if not path.exists():
+            raise OSError(f"Path {path} does not exist.")
+        new_path = self._get_path(new_name)
+        if path.exists():
+            raise OSError(f"Path {path} already exists.")
+
+        meta_path = self._get_meta_path(name)
+        if not path.exists():
+            raise OSError(f"Path {meta_path} does not exist.")
+        new_meta_path = self._get_meta_path(name)
+        if path.exists():
+            raise OSError(f"Path {path} already exists.")
+
+        path.rename(new_path)
+        meta_path.rename(new_meta_path)
+
+    def list_files(self):
+        return [x for x in self._files_directory.iterdir() if x.is_file()]

tests/conftest.py

@@ -2,22 +2,50 @@ import pytest
 import yaml
 from sachet.server.users import manage
 from click.testing import CliRunner
-from sachet.server import app, db
+from sachet.server import app, db, storage
 from sachet.server.models import Permissions, User
 from bitmask import Bitmask
+from pathlib import Path
+import random
+import itertools
 
 
 @pytest.fixture
-def client():
+def rand():
+    """Deterministic random data generator.
+
+    Be sure to seed 0 with each test!
+    """
+    r = random.Random()
+    r.seed(0)
+    return r
+
+
+@pytest.fixture
+def client(config={}):
     """Flask application with DB already set up and ready."""
     with app.test_client() as client:
         with app.app_context():
+            for k, v in config.items():
+                app.config[k] = v
+
             db.drop_all()
             db.create_all()
             db.session.commit()
             yield client
             db.session.remove()
             db.drop_all()
+            if app.config["SACHET_STORAGE"] == "filesystem":
+                for file in itertools.chain(
+                    storage._meta_directory.iterdir(),
+                    storage._files_directory.iterdir(),
+                ):
+                    if file.is_relative_to(Path(app.instance_path)) and file.is_file():
+                        file.unlink()
+                    else:
+                        raise OSError(
+                            f"Attempted to delete {file}: please delete it yourself."
+                        )
 
 
 @pytest.fixture

tests/test_storage.py

@@ -0,0 +1,40 @@
+import pytest
+
+from sachet.server import storage
+from uuid import UUID
+
+"""Test suite for storage backends (not their API endpoints)."""
+
+
+# if other storage backends are implemented we test them with the same suite with this line
+@pytest.mark.parametrize("client", [{"SACHET_STORAGE": "filesystem"}], indirect=True)
+class TestSuite:
+    def test_creation(self, client, rand):
+        """Test the process of creating, writing, then reading files with metadata."""
+
+        files = [
+            dict(
+                name=str(UUID(bytes=rand.randbytes(16))),
+                data=rand.randbytes(4000),
+                metadata=dict(
+                    sdljkf=dict(abc="def", aaa="bbb"),
+                    lkdsjf=dict(ld="sdlfj", sdljf="sdlkjf"),
+                    sdlfkj="sjdlkfsldk",
+                    ssjdklf=rand.randint(-1000, 1000),
+                ),
+            )
+            for i in range(25)
+        ]
+
+        for file in files:
+            storage.create(file["name"])
+            with storage.open(file["name"], mode="wb") as f:
+                f.write(file["data"])
+            storage.write_metadata(file["name"], file["metadata"])
+
+        for file in files:
+            with storage.open(file["name"], mode="rb") as f:
+                saved_data = f.read()
+                assert saved_data == file["data"]
+                saved_meta = storage.read_metadata(file["name"])
+                assert saved_meta == file["metadata"]