diff --git a/requirements.txt b/requirements.txt index 6de3da3..7afb69f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,6 +31,7 @@ pytest-env==0.8.1 PyYAML==6.0 six==1.16.0 SQLAlchemy==2.0.5.post1 +SQLAlchemy-Utils==0.40.0 tomli==2.0.1 typing_extensions==4.5.0 Werkzeug==2.2.3 diff --git a/sachet/server/__init__.py b/sachet/server/__init__.py index 83c2949..8f411b7 100644 --- a/sachet/server/__init__.py +++ b/sachet/server/__init__.py @@ -42,5 +42,9 @@ from sachet.server.admin.views import admin_blueprint app.register_blueprint(admin_blueprint) +from sachet.server.files.views import files_blueprint + +app.register_blueprint(files_blueprint) + with app.app_context(): db.create_all() diff --git a/sachet/server/files/views.py b/sachet/server/files/views.py index f16d655..05d4ca9 100644 --- a/sachet/server/files/views.py +++ b/sachet/server/files/views.py @@ -1,8 +1,10 @@ -import jwt -from flask import Blueprint, request, jsonify +import uuid +import io +from flask import Blueprint, request, jsonify, send_file from flask.views import MethodView -from sachet.server.models import File, Permissions +from sachet.server.models import Share, Permissions from sachet.server.views_common import ModelAPI, auth_required +from sachet.server import storage, db files_blueprint = Blueprint("files_blueprint", __name__) @@ -10,26 +12,156 @@ files_blueprint = Blueprint("files_blueprint", __name__) class FilesAPI(ModelAPI): """Files metadata API.""" - @auth_required - def get(self, id, auth_user=None): - pass + @auth_required(required_permissions=(Permissions.READ,)) + def get(self, share_id, auth_user=None): + share = Share.query.filter_by(share_id=share_id).first() + return super().get(share) + + @auth_required(required_permissions=(Permissions.MODIFY,)) + def patch(self, share_id, auth_user=None): + share = Share.query.filter_by(share_id=share_id).first() + return super().patch(share) + + @auth_required(required_permissions=(Permissions.MODIFY,)) + def put(self, share_id, auth_user=None): + share = Share.query.filter_by(share_id=share_id).first() + return super().put(share) + + @auth_required(required_permissions=(Permissions.DELETE,)) + def delete(self, share_id, auth_user=None): + share = Share.query.filter_by(share_id=share_id).first() + return super().delete(share) files_blueprint.add_url_rule( - "/files/", + "/files/", view_func=FilesAPI.as_view("files_api"), - methods=["POST", "PUT", "PATCH", "GET", "DELETE"], + methods=["PUT", "PATCH", "GET", "DELETE"], ) + +class FileCreationAPI(ModelAPI): + @auth_required(required_permissions=(Permissions.CREATE,)) + def post(self, auth_user=None): + # silent means it will return None if there is no JSON + data = request.get_json(silent=True) or {} + data["owner_name"] = auth_user.username + return super().post(Share, data) + + files_blueprint.add_url_rule( - "/files//content", - view_func=FilesContentAPI.as_view("files_content_api"), - methods=["PUT", "GET"], + "/files", + view_func=FileCreationAPI.as_view("files_creation_api"), + methods=["POST"], ) -users_blueprint.add_url_rule( - "/users/", - view_func=UserAPI.as_view("user_api"), - methods=["GET", "PATCH", "PUT"], +class FileContentAPI(ModelAPI): + @auth_required(required_permissions=(Permissions.CREATE,)) + def post(self, share_id, auth_user=None): + share = Share.query.filter_by(share_id=uuid.UUID(share_id)).first() + + if not share: + return ( + jsonify({"status": "fail", "message": "This share does not exist."}) + ), 404 + + if auth_user != share.owner: + return ( + jsonify( + { + "status": "fail", + "message": "Share must be initialized by its owner.", + } + ), + 403, + ) + + if share.initialized: + return ( + jsonify( + { + "status": "fail", + "message": "Share already initialized. Use PUT to modify the share.", + } + ), + 423, + ) + + upload = request.files["upload"] + data = upload.read() + file = share.get_handle() + with file.open(mode="wb") as f: + f.write(data) + + share.initialized = True + + db.session.commit() + + return ( + jsonify({"status": "success", "message": "Share has been initialized."}), + 201, + ) + + @auth_required(required_permissions=(Permissions.MODIFY,)) + def put(self, share_id, auth_user=None): + share = Share.query.filter_by(share_id=share_id).first() + if not share: + return ( + jsonify({"status": "fail", "message": "This share does not exist."}) + ), 404 + + if not share.initialized: + return ( + jsonify( + { + "status": "fail", + "message": "Share not initialized. Use POST to upload for the first time to this share.", + } + ), + 423, + ) + + upload = request.files["upload"] + data = upload.read() + file = share.get_handle() + + with file.open(mode="wb") as f: + f.write(data) + + return ( + jsonify({"status": "success", "message": "Share has been modified."}), + 200, + ) + + @auth_required(required_permissions=(Permissions.READ,)) + def get(self, share_id, auth_user=None): + share = Share.query.filter_by(share_id=share_id).first() + if not share: + return ( + jsonify({"status": "fail", "message": "This share does not exist."}) + ), 404 + + if not share.initialized: + return ( + jsonify( + { + "status": "fail", + "message": "Share not initialized. Use POST to upload for the first time to this share.", + } + ), + 404, + ) + + file = share.get_handle() + with file.open(mode="rb") as f: + data = f.read() + + return send_file(io.BytesIO(data), download_name=str(share.share_id)) + + +files_blueprint.add_url_rule( + "/files//content", + view_func=FileContentAPI.as_view("files_content_api"), + methods=["POST", "PUT", "GET"], ) diff --git a/sachet/server/models.py b/sachet/server/models.py index 931334b..c4acc9f 100644 --- a/sachet/server/models.py +++ b/sachet/server/models.py @@ -1,11 +1,12 @@ -from sachet.server import app, db, ma, bcrypt -from marshmallow import fields, ValidationError -from flask import request, jsonify -from functools import wraps +from sachet.server import app, db, ma, bcrypt, storage import datetime import jwt -from bitmask import Bitmask from enum import IntFlag +from bitmask import Bitmask +from marshmallow import fields, ValidationError +from flask import request, jsonify, url_for +from sqlalchemy_utils import UUIDType +import uuid class Permissions(IntFlag): @@ -15,6 +16,7 @@ class Permissions(IntFlag): LOCK = 1 << 3 LIST = 1 << 4 ADMIN = 1 << 5 + READ = 1 << 6 class PermissionField(fields.Field): @@ -188,3 +190,63 @@ class ServerSettings(db.Model): default_permissions = PermissionField() return Schema() + + +class Share(db.Model): + """Share for a single file. + + Parameters + ---------- + owner : User + Assign this share to this user. + + Attributes + ---------- + share_id : uuid.uuid4 + Unique identifier for this given share. + owner : User + The user who owns this share. + initialized : bool + Since only the metadata is uploaded first, this switches to True when + the real data is uploaded. + create_date : DateTime + Time the share was created (not initialized.) + + Methods + ------- + get_handle(): + Obtain a sachet.storage.Storage.File handle. This can be used to modify + the file contents. + """ + + __tablename__ = "shares" + + share_id = db.Column(UUIDType(), primary_key=True, default=uuid.uuid4) + + owner_name = db.Column(db.String, db.ForeignKey("users.username")) + owner = db.relationship("User", backref=db.backref("owner")) + + initialized = db.Column(db.Boolean, nullable=False, default=False) + + create_date = db.Column(db.DateTime, nullable=False) + + def __init__(self, owner_name): + self.owner = User.query.filter_by(username=owner_name).first() + self.owner_name = self.owner.username + self.share_id = uuid.uuid4() + self.url = url_for("files_blueprint.files_api", share_id=self.share_id) + self.create_date = datetime.datetime.now() + + def get_schema(self): + class Schema(ma.SQLAlchemySchema): + class Meta: + model = self + + share_id = ma.auto_field(dump_only=True) + owner_name = ma.auto_field() + initialized = ma.auto_field(dump_only=True) + + return Schema() + + def get_handle(self): + return storage.get_file(str(self.share_id)) diff --git a/sachet/server/views_common.py b/sachet/server/views_common.py index f59ce3f..52453a6 100644 --- a/sachet/server/views_common.py +++ b/sachet/server/views_common.py @@ -166,17 +166,25 @@ class ModelAPI(MethodView): } return jsonify(resp), 404 - model.delete() + db.session.delete(model) db.session.commit() return jsonify({"status": "success"}) - def post(self, ModelClass, data): - model_schema = ModelClass.get_schema() + def post(self, ModelClass, data={}): + """Create new instance of a class. + + Parameters + ---------- + ModelClass + Class to make an instance of. + data : dict + Object that can be loaded with Marshmallow to create the class. + """ + model_schema = ModelClass.get_schema(ModelClass) - post_json = request.get_json() try: - deserialized = model_schema.load(post_json) + deserialized = model_schema.load(data) except ValidationError as e: resp = {"status": "fail", "message": f"Invalid data: {str(e)}"} return jsonify(resp), 400 @@ -184,4 +192,7 @@ class ModelAPI(MethodView): # create new ModelClass instance with all the parameters given in the request model = ModelClass(**deserialized) - return jsonify({"status": "success"}), 201 + db.session.add(model) + db.session.commit() + + return jsonify({"status": "success", "url": model.url}), 201 diff --git a/sachet/storage/__init__.py b/sachet/storage/__init__.py index 3cd7df6..e1d8bc5 100644 --- a/sachet/storage/__init__.py +++ b/sachet/storage/__init__.py @@ -33,14 +33,12 @@ class Storage: pass class File: - """Handle for a file and its metadata. + """Handle for a file. Do not instantiate this; use `Storage.get_file()`. Attributes ---------- - metadata : _Metadata - All the metadata, accessible via dot attribute notation. name : str Filename """ @@ -63,7 +61,7 @@ class Storage: pass def delete(self): - """Delete file and associated metadata.""" + """Delete file.""" pass diff --git a/sachet/storage/filesystem.py b/sachet/storage/filesystem.py index 086eb5e..613f8a5 100644 --- a/sachet/storage/filesystem.py +++ b/sachet/storage/filesystem.py @@ -16,10 +16,8 @@ class FileSystem(Storage): self._directory = Path(app.instance_path) / config_path self._files_directory = self._directory / Path("files") - self._meta_directory = self._directory / Path("meta") self._files_directory.mkdir(mode=0o700, exist_ok=True, parents=True) - self._meta_directory.mkdir(mode=0o700, exist_ok=True, parents=True) if not self._directory.is_dir(): raise OSError(f"'{app.config['SACHET_FILE_DIR']}' is not a directory.") @@ -28,10 +26,6 @@ class FileSystem(Storage): name = secure_filename(name) return self._files_directory / Path(name) - def _get_meta_path(self, name): - name = secure_filename(name) - return self._meta_directory / Path(name) - def list_files(self): return [ self.get_file(x.name) @@ -47,14 +41,10 @@ class FileSystem(Storage): self.name = name self._storage = storage self._path = self._storage._get_path(name) - self._meta_path = self._storage._get_meta_path(name) self._path.touch() - self._meta_path.touch() - self.metadata = self._Metadata(self) def delete(self, name): self._path.unlink() - self._meta_path.unlink() def open(self, mode="r"): return self._path.open(mode=mode) @@ -64,33 +54,4 @@ class FileSystem(Storage): if new_path.exists(): raise OSError(f"Path {path} already exists.") - new_meta_path = self._storage._get_meta_path(new_name) - if new_meta_path.exists(): - raise OSError(f"Path {path} already exists.") - self._path.rename(new_path) - self._meta_path.rename(new_meta_path) - - class _Metadata: - def __init__(self, file): - self.__dict__["_file"] = file - - @property - def __data(self): - with self._file._meta_path.open() as meta_file: - content = meta_file.read() - if len(content.strip()) == 0: - return {} - return json.loads(content) - - # there is no setter for __data because it would cause __setattr__ to infinitely recurse - - def __setattr__(self, name, value): - data = self.__data - data[name] = value - with self._file._meta_path.open("w") as meta_file: - content = json.dumps(data) - meta_file.write(content) - - def __getattr__(self, name): - return self.__data.get(name, None) diff --git a/tests/conftest.py b/tests/conftest.py index 368f220..28323cd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,4 @@ import pytest -import yaml from sachet.server.users import manage from click.testing import CliRunner from sachet.server import app, db, storage @@ -7,7 +6,6 @@ from sachet.server.models import Permissions, User from bitmask import Bitmask from pathlib import Path import random -import itertools @pytest.fixture @@ -23,10 +21,7 @@ def rand(): def clear_filesystem(): if app.config["SACHET_STORAGE"] == "filesystem": - for file in itertools.chain( - storage._meta_directory.iterdir(), - storage._files_directory.iterdir(), - ): + for file in storage._files_directory.iterdir(): if file.is_relative_to(Path(app.instance_path)) and file.is_file(): file.unlink() else: @@ -67,7 +62,12 @@ def users(client): Returns a dictionary with all the info for each user. """ userinfo = dict( - jeff=dict(password="1234", permissions=Bitmask()), + jeff=dict( + password="1234", + permissions=Bitmask( + Permissions.CREATE, Permissions.READ, Permissions.DELETE + ), + ), administrator=dict(password="4321", permissions=Bitmask(Permissions.ADMIN)), ) diff --git a/tests/test_files.py b/tests/test_files.py new file mode 100644 index 0000000..a877ed5 --- /dev/null +++ b/tests/test_files.py @@ -0,0 +1,68 @@ +import pytest +from io import BytesIO +from werkzeug.datastructures import FileStorage + +"""Test file share endpoints.""" + + +# if other storage backends are implemented we test them with the same suite +# this might be redundant because test_storage tests the backends already +@pytest.mark.parametrize("client", [{"SACHET_STORAGE": "filesystem"}], indirect=True) +class TestSuite: + def test_sharing(self, client, users, tokens, rand): + # create share + resp = client.post( + "/files", headers={"Authorization": f"bearer {tokens['jeff']}"} + ) + assert resp.status_code == 201 + + data = resp.get_json() + url = data.get("url") + + assert url is not None + assert "/files/" in url + + upload_data = rand.randbytes(4000) + + # upload file to share + resp = client.post( + url + "/content", + headers={"Authorization": f"bearer {tokens['jeff']}"}, + data={ + "upload": FileStorage(stream=BytesIO(upload_data), filename="upload") + }, + content_type="multipart/form-data", + ) + assert resp.status_code == 201 + + # test not allowing re-upload + resp = client.post( + url + "/content", + headers={"Authorization": f"bearer {tokens['jeff']}"}, + data={ + "upload": FileStorage(stream=BytesIO(upload_data), filename="upload") + }, + content_type="multipart/form-data", + ) + assert resp.status_code == 423 + + # read file + resp = client.get( + url + "/content", + headers={"Authorization": f"bearer {tokens['jeff']}"}, + ) + assert resp.data == upload_data + + # test deletion + resp = client.delete( + url, + headers={"Authorization": f"bearer {tokens['jeff']}"}, + ) + assert resp.status_code == 200 + + # file shouldn't exist anymore + resp = client.get( + url + "/content", + headers={"Authorization": f"bearer {tokens['jeff']}"}, + ) + assert resp.status_code == 404 diff --git a/tests/test_storage.py b/tests/test_storage.py index c554887..88d4230 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -10,18 +10,12 @@ from uuid import UUID @pytest.mark.parametrize("client", [{"SACHET_STORAGE": "filesystem"}], indirect=True) class TestSuite: def test_creation(self, client, rand): - """Test the process of creating, writing, then reading files with metadata, and also listing files.""" + """Test the process of creating, writing, then reading files, and also listing files.""" files = [ dict( name=str(UUID(bytes=rand.randbytes(16))), data=rand.randbytes(4000), - metadata=dict( - sdljkf=dict(abc="def", aaa="bbb"), - lkdsjf=dict(ld="sdlfj", sdljf="sdlkjf"), - sdlfkj="sjdlkfsldk", - ssjdklf=rand.randint(-1000, 1000), - ), ) for i in range(25) ] @@ -30,15 +24,12 @@ class TestSuite: handle = storage.get_file(file["name"]) with handle.open(mode="wb") as f: f.write(file["data"]) - handle.metadata.test_data = file["metadata"] for file in files: handle = storage.get_file(file["name"]) with handle.open(mode="rb") as f: saved_data = f.read() assert saved_data == file["data"] - saved_meta = handle.metadata.test_data - assert saved_meta == file["metadata"] assert sorted([f.name for f in storage.list_files()]) == sorted( [f["name"] for f in files] @@ -50,12 +41,6 @@ class TestSuite: name=str(UUID(bytes=rand.randbytes(16))), new_name=str(UUID(bytes=rand.randbytes(16))), data=rand.randbytes(4000), - metadata=dict( - sdljkf=dict(abc="def", aaa="bbb"), - lkdsjf=dict(ld="sdlfj", sdljf="sdlkjf"), - sdlfkj="sjdlkfsldk", - ssjdklf=rand.randint(-1000, 1000), - ), ) for i in range(25) ] @@ -64,7 +49,6 @@ class TestSuite: handle = storage.get_file(file["name"]) with handle.open(mode="wb") as f: f.write(file["data"]) - handle.metadata.test_data = file["metadata"] handle.rename(file["new_name"]) for file in files: @@ -72,5 +56,3 @@ class TestSuite: with handle.open(mode="rb") as f: saved_data = f.read() assert saved_data == file["data"] - saved_meta = handle.metadata.test_data - assert saved_meta == file["metadata"]