From 95ed51dd64369d06435602f7f131406a151b30c9 Mon Sep 17 00:00:00 2001 From: dogeystamp Date: Sat, 8 Apr 2023 18:35:56 -0400 Subject: [PATCH] storage: implemented filesystem test coverage to be improved --- config.yml.example | 2 + sachet/server/__init__.py | 11 +++++ sachet/server/config.py | 4 ++ sachet/storage/__init__.py | 71 +++++++++++++++++++++++++++ sachet/storage/filesystem.py | 95 ++++++++++++++++++++++++++++++++++++ tests/conftest.py | 32 +++++++++++- tests/test_storage.py | 40 +++++++++++++++ 7 files changed, 253 insertions(+), 2 deletions(-) create mode 100644 sachet/storage/__init__.py create mode 100644 sachet/storage/filesystem.py create mode 100644 tests/test_storage.py diff --git a/config.yml.example b/config.yml.example index 6b14784..eddbc05 100644 --- a/config.yml.example +++ b/config.yml.example @@ -3,3 +3,5 @@ SECRET_KEY: "" # BCRYPT_LOG_ROUNDS: 13 +# SACHET_STORAGE: "filesystem" +# SACHET_FILE_DIR: "/srv/sachet/storage" diff --git a/sachet/server/__init__.py b/sachet/server/__init__.py index 7edd62b..83c2949 100644 --- a/sachet/server/__init__.py +++ b/sachet/server/__init__.py @@ -21,6 +21,17 @@ bcrypt = Bcrypt(app) db = SQLAlchemy(app) ma = Marshmallow() +_storage_method = app.config["SACHET_STORAGE"] + +storage = None + +from sachet.storage import FileSystem + +if _storage_method == "filesystem": + storage = FileSystem() +else: + raise ValueError(f"{_storage_method} is not a valid storage method.") + import sachet.server.commands from sachet.server.users.views import users_blueprint diff --git a/sachet/server/config.py b/sachet/server/config.py index bb72653..e5f3a8e 100644 --- a/sachet/server/config.py +++ b/sachet/server/config.py @@ -8,16 +8,20 @@ class BaseConfig: SQLALCHEMY_DATABASE_URI = sqlalchemy_base + ".db" BCRYPT_LOG_ROUNDS = 13 SQLALCHEMY_TRACK_MODIFICATIONS = False + SACHET_STORAGE = "filesystem" + SACHET_FILE_DIR = "/srv/sachet/storage" class TestingConfig(BaseConfig): SQLALCHEMY_DATABASE_URI = sqlalchemy_base + "_test" + ".db" BCRYPT_LOG_ROUNDS = 4 + SACHET_FILE_DIR = "storage_test" class DevelopmentConfig(BaseConfig): SQLALCHEMY_DATABASE_URI = sqlalchemy_base + "_dev" + ".db" BCRYPT_LOG_ROUNDS = 4 + SACHET_FILE_DIR = "storage_dev" class ProductionConfig(BaseConfig): diff --git a/sachet/storage/__init__.py b/sachet/storage/__init__.py new file mode 100644 index 0000000..416584c --- /dev/null +++ b/sachet/storage/__init__.py @@ -0,0 +1,71 @@ +class Storage: + """Generic storage interface. + + Raises: + OSError if SACHET_FILE_DIR could not be opened as a directory. + """ + + def create(self, name): + """Create file along with the metadata. + + Raises: + OSError if the file already exists. + """ + + pass + + def open(self, name, mode="r"): + """Open a file for reading/writing. + + Raises: + OSError if the file does not exist. + + Returns: + Stream to access file (similar to open()'s handle.) + """ + + pass + + def read_metadata(self, name): + """Get metadata for a file. + + Raises: + OSError if the file does not exist. + """ + + pass + + def write_metadata(self, name, data): + """Set metadata for a file. + + Raises: + OSError if the file does not exist. + """ + + pass + + def delete(self, name): + """Delete file and associated metadata. + + Raises: + OSError if the file does not exist. + """ + + pass + + def rename(self, name, new_name): + """Rename a file. + + Raises: + OSError if the file does not exist. + """ + + pass + + def list_files(self): + """Lists all files.""" + + pass + + +from .filesystem import FileSystem diff --git a/sachet/storage/filesystem.py b/sachet/storage/filesystem.py new file mode 100644 index 0000000..219cac4 --- /dev/null +++ b/sachet/storage/filesystem.py @@ -0,0 +1,95 @@ +from sachet.storage import Storage +from pathlib import Path +from sachet.server import app +from werkzeug.utils import secure_filename +import json + + +class FileSystem(Storage): + def __init__(self): + config_path = Path(app.config["SACHET_FILE_DIR"]) + if config_path.is_absolute(): + self._directory = config_path + else: + self._directory = Path(app.instance_path) / config_path + + self._files_directory = self._directory / Path("files") + self._meta_directory = self._directory / Path("meta") + + self._files_directory.mkdir(mode=0o700, exist_ok=True, parents=True) + self._meta_directory.mkdir(mode=0o700, exist_ok=True, parents=True) + + if not self._directory.is_dir(): + raise OSError(f"'{app.config['SACHET_FILE_DIR']}' is not a directory.") + + def _get_path(self, name): + name = secure_filename(name) + return self._files_directory / Path(name) + + def _get_meta_path(self, name): + name = secure_filename(name) + return self._meta_directory / Path(name) + + def create(self, name): + path = self._get_path(name) + if path.exists(): + raise OSError(f"Path {path} already exists.") + + meta_path = self._get_meta_path(name) + if meta_path.exists(): + raise OSError(f"Path {meta_path} already exists.") + + path.touch() + meta_path.touch() + + def delete(self, name): + path = self._get_path(name) + if not path.exists(): + raise OSError(f"Path {path} does not exist.") + path.unlink() + + def open(self, name, mode="r"): + path = self._get_path(name) + if not path.exists(): + raise OSError(f"Path {path} does not exist.") + + return path.open(mode=mode) + + def read_metadata(self, name): + meta_path = self._get_meta_path(name) + if not meta_path.exists(): + raise OSError(f"Path {meta_path} does not exist.") + + with meta_path.open() as meta_file: + content = meta_file.read() + return json.loads(content) + + def write_metadata(self, name, data): + meta_path = self._get_meta_path(name) + if not meta_path.exists(): + raise OSError(f"Path {meta_path} does not exist.") + + with meta_path.open("w") as meta_file: + content = json.dumps(data) + meta_file.write(content) + + def rename(self, name, new_name): + path = self._get_path(name) + if not path.exists(): + raise OSError(f"Path {path} does not exist.") + new_path = self._get_path(new_name) + if path.exists(): + raise OSError(f"Path {path} already exists.") + + meta_path = self._get_meta_path(name) + if not path.exists(): + raise OSError(f"Path {meta_path} does not exist.") + new_meta_path = self._get_meta_path(name) + if path.exists(): + raise OSError(f"Path {path} already exists.") + + path.rename(new_path) + meta_path.rename(new_meta_path) + + def list_files(self): + return [x for x in self._files_directory.iterdir() if x.is_file()] diff --git a/tests/conftest.py b/tests/conftest.py index d68e41c..e4b215d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,22 +2,50 @@ import pytest import yaml from sachet.server.users import manage from click.testing import CliRunner -from sachet.server import app, db +from sachet.server import app, db, storage from sachet.server.models import Permissions, User from bitmask import Bitmask +from pathlib import Path +import random +import itertools @pytest.fixture -def client(): +def rand(): + """Deterministic random data generator. + + Be sure to seed 0 with each test! + """ + r = random.Random() + r.seed(0) + return r + + +@pytest.fixture +def client(config={}): """Flask application with DB already set up and ready.""" with app.test_client() as client: with app.app_context(): + for k, v in config.items(): + app.config[k] = v + db.drop_all() db.create_all() db.session.commit() yield client db.session.remove() db.drop_all() + if app.config["SACHET_STORAGE"] == "filesystem": + for file in itertools.chain( + storage._meta_directory.iterdir(), + storage._files_directory.iterdir(), + ): + if file.is_relative_to(Path(app.instance_path)) and file.is_file(): + file.unlink() + else: + raise OSError( + f"Attempted to delete {file}: please delete it yourself." + ) @pytest.fixture diff --git a/tests/test_storage.py b/tests/test_storage.py new file mode 100644 index 0000000..aa9f4e7 --- /dev/null +++ b/tests/test_storage.py @@ -0,0 +1,40 @@ +import pytest + +from sachet.server import storage +from uuid import UUID + +"""Test suite for storage backends (not their API endpoints).""" + + +# if other storage backends are implemented we test them with the same suite with this line +@pytest.mark.parametrize("client", [{"SACHET_STORAGE": "filesystem"}], indirect=True) +class TestSuite: + def test_creation(self, client, rand): + """Test the process of creating, writing, then reading files with metadata.""" + + files = [ + dict( + name=str(UUID(bytes=rand.randbytes(16))), + data=rand.randbytes(4000), + metadata=dict( + sdljkf=dict(abc="def", aaa="bbb"), + lkdsjf=dict(ld="sdlfj", sdljf="sdlkjf"), + sdlfkj="sjdlkfsldk", + ssjdklf=rand.randint(-1000, 1000), + ), + ) + for i in range(25) + ] + + for file in files: + storage.create(file["name"]) + with storage.open(file["name"], mode="wb") as f: + f.write(file["data"]) + storage.write_metadata(file["name"], file["metadata"]) + + for file in files: + with storage.open(file["name"], mode="rb") as f: + saved_data = f.read() + assert saved_data == file["data"] + saved_meta = storage.read_metadata(file["name"]) + assert saved_meta == file["metadata"]