sachet/sachet-server
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

/files/: implemented endpoint

Browse Source
This commit is contained in:
dogeystamp 2023-04-10 22:17:12 -04:00
parent 2fba574773
commit af21887402
Signed by: dogeystamp
GPG Key ID: 7225FE3592EFFA38
10 changed files with 314 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
requirements.txt
View File

@ -31,6 +31,7 @@ pytest-env==0.8.1
PyYAML==6.0
six==1.16.0
SQLAlchemy==2.0.5.post1
SQLAlchemy-Utils==0.40.0
tomli==2.0.1
typing_extensions==4.5.0
Werkzeug==2.2.3

4
sachet/server/__init__.py
View File

@ -42,5 +42,9 @@ from sachet.server.admin.views import admin_blueprint
app.register_blueprint(admin_blueprint)
from sachet.server.files.views import files_blueprint
app.register_blueprint(files_blueprint)
with app.app_context():
db.create_all()

162
sachet/server/files/views.py
View File

@ -1,8 +1,10 @@
import jwt
from flask import Blueprint, request, jsonify
import uuid
import io
from flask import Blueprint, request, jsonify, send_file
from flask.views import MethodView
from sachet.server.models import File, Permissions
from sachet.server.models import Share, Permissions
from sachet.server.views_common import ModelAPI, auth_required
from sachet.server import storage, db
files_blueprint = Blueprint("files_blueprint", __name__)
@ -10,26 +12,156 @@ files_blueprint = Blueprint("files_blueprint", __name__)
class FilesAPI(ModelAPI):
"""Files metadata API."""
@auth_required
def get(self, id, auth_user=None):
pass
@auth_required(required_permissions=(Permissions.READ,))
def get(self, share_id, auth_user=None):
share = Share.query.filter_by(share_id=share_id).first()
return super().get(share)
@auth_required(required_permissions=(Permissions.MODIFY,))
def patch(self, share_id, auth_user=None):
share = Share.query.filter_by(share_id=share_id).first()
return super().patch(share)
@auth_required(required_permissions=(Permissions.MODIFY,))
def put(self, share_id, auth_user=None):
share = Share.query.filter_by(share_id=share_id).first()
return super().put(share)
@auth_required(required_permissions=(Permissions.DELETE,))
def delete(self, share_id, auth_user=None):
share = Share.query.filter_by(share_id=share_id).first()
return super().delete(share)
files_blueprint.add_url_rule(
"/files/<id>",
"/files/<share_id>",
view_func=FilesAPI.as_view("files_api"),
methods=["POST", "PUT", "PATCH", "GET", "DELETE"],
methods=["PUT", "PATCH", "GET", "DELETE"],
)
class FileCreationAPI(ModelAPI):
@auth_required(required_permissions=(Permissions.CREATE,))
def post(self, auth_user=None):
# silent means it will return None if there is no JSON
data = request.get_json(silent=True) or {}
data["owner_name"] = auth_user.username
return super().post(Share, data)
files_blueprint.add_url_rule(
"/files/<id>/content",
view_func=FilesContentAPI.as_view("files_content_api"),
methods=["PUT", "GET"],
"/files",
view_func=FileCreationAPI.as_view("files_creation_api"),
methods=["POST"],
)
users_blueprint.add_url_rule(
"/users/<username>",
view_func=UserAPI.as_view("user_api"),
methods=["GET", "PATCH", "PUT"],
class FileContentAPI(ModelAPI):
@auth_required(required_permissions=(Permissions.CREATE,))
def post(self, share_id, auth_user=None):
share = Share.query.filter_by(share_id=uuid.UUID(share_id)).first()
if not share:
return (
jsonify({"status": "fail", "message": "This share does not exist."})
), 404
if auth_user != share.owner:
return (
jsonify(
{
"status": "fail",
"message": "Share must be initialized by its owner.",
}
),
403,
)
if share.initialized:
return (
jsonify(
{
"status": "fail",
"message": "Share already initialized. Use PUT to modify the share.",
}
),
423,
)
upload = request.files["upload"]
data = upload.read()
file = share.get_handle()
with file.open(mode="wb") as f:
f.write(data)
share.initialized = True
db.session.commit()
return (
jsonify({"status": "success", "message": "Share has been initialized."}),
201,
)
@auth_required(required_permissions=(Permissions.MODIFY,))
def put(self, share_id, auth_user=None):
share = Share.query.filter_by(share_id=share_id).first()
if not share:
return (
jsonify({"status": "fail", "message": "This share does not exist."})
), 404
if not share.initialized:
return (
jsonify(
{
"status": "fail",
"message": "Share not initialized. Use POST to upload for the first time to this share.",
}
),
423,
)
upload = request.files["upload"]
data = upload.read()
file = share.get_handle()
with file.open(mode="wb") as f:
f.write(data)
return (
jsonify({"status": "success", "message": "Share has been modified."}),
200,
)
@auth_required(required_permissions=(Permissions.READ,))
def get(self, share_id, auth_user=None):
share = Share.query.filter_by(share_id=share_id).first()
if not share:
return (
jsonify({"status": "fail", "message": "This share does not exist."})
), 404
if not share.initialized:
return (
jsonify(
{
"status": "fail",
"message": "Share not initialized. Use POST to upload for the first time to this share.",
}
),
404,
)
file = share.get_handle()
with file.open(mode="rb") as f:
data = f.read()
return send_file(io.BytesIO(data), download_name=str(share.share_id))
files_blueprint.add_url_rule(
"/files/<share_id>/content",
view_func=FileContentAPI.as_view("files_content_api"),
methods=["POST", "PUT", "GET"],
)

72
sachet/server/models.py
View File

@ -1,11 +1,12 @@
from sachet.server import app, db, ma, bcrypt
from marshmallow import fields, ValidationError
from flask import request, jsonify
from functools import wraps
from sachet.server import app, db, ma, bcrypt, storage
import datetime
import jwt
from bitmask import Bitmask
from enum import IntFlag
from bitmask import Bitmask
from marshmallow import fields, ValidationError
from flask import request, jsonify, url_for
from sqlalchemy_utils import UUIDType
import uuid
class Permissions(IntFlag):
@ -15,6 +16,7 @@ class Permissions(IntFlag):
LOCK = 1 << 3
LIST = 1 << 4
ADMIN = 1 << 5
READ = 1 << 6
class PermissionField(fields.Field):
@ -188,3 +190,63 @@ class ServerSettings(db.Model):
default_permissions = PermissionField()
return Schema()
class Share(db.Model):
"""Share for a single file.
Parameters
----------
owner : User
Assign this share to this user.
Attributes
----------
share_id : uuid.uuid4
Unique identifier for this given share.
owner : User
The user who owns this share.
initialized : bool
Since only the metadata is uploaded first, this switches to True when
the real data is uploaded.
create_date : DateTime
Time the share was created (not initialized.)
Methods
-------
get_handle():
Obtain a sachet.storage.Storage.File handle. This can be used to modify
the file contents.
"""
__tablename__ = "shares"
share_id = db.Column(UUIDType(), primary_key=True, default=uuid.uuid4)
owner_name = db.Column(db.String, db.ForeignKey("users.username"))
owner = db.relationship("User", backref=db.backref("owner"))
initialized = db.Column(db.Boolean, nullable=False, default=False)
create_date = db.Column(db.DateTime, nullable=False)
def __init__(self, owner_name):
self.owner = User.query.filter_by(username=owner_name).first()
self.owner_name = self.owner.username
self.share_id = uuid.uuid4()
self.url = url_for("files_blueprint.files_api", share_id=self.share_id)
self.create_date = datetime.datetime.now()
def get_schema(self):
class Schema(ma.SQLAlchemySchema):
class Meta:
model = self
share_id = ma.auto_field(dump_only=True)
owner_name = ma.auto_field()
initialized = ma.auto_field(dump_only=True)
return Schema()
def get_handle(self):
return storage.get_file(str(self.share_id))

23
sachet/server/views_common.py
View File

@ -166,17 +166,25 @@ class ModelAPI(MethodView):
}
return jsonify(resp), 404
model.delete()
db.session.delete(model)
db.session.commit()
return jsonify({"status": "success"})
def post(self, ModelClass, data):
model_schema = ModelClass.get_schema()
def post(self, ModelClass, data={}):
"""Create new instance of a class.
Parameters
----------
ModelClass
Class to make an instance of.
data : dict
Object that can be loaded with Marshmallow to create the class.
"""
model_schema = ModelClass.get_schema(ModelClass)
post_json = request.get_json()
try:
deserialized = model_schema.load(post_json)
deserialized = model_schema.load(data)
except ValidationError as e:
resp = {"status": "fail", "message": f"Invalid data: {str(e)}"}
return jsonify(resp), 400
@ -184,4 +192,7 @@ class ModelAPI(MethodView):
# create new ModelClass instance with all the parameters given in the request
model = ModelClass(**deserialized)
return jsonify({"status": "success"}), 201
db.session.add(model)
db.session.commit()
return jsonify({"status": "success", "url": model.url}), 201

6
sachet/storage/__init__.py
View File

@ -33,14 +33,12 @@ class Storage:
pass
class File:
"""Handle for a file and its metadata.
"""Handle for a file.
Do not instantiate this; use `Storage.get_file()`.
Attributes
----------
metadata : _Metadata
All the metadata, accessible via dot attribute notation.
name : str
Filename
"""
@ -63,7 +61,7 @@ class Storage:
pass
def delete(self):
"""Delete file and associated metadata."""
"""Delete file."""
pass

39
sachet/storage/filesystem.py
View File

@ -16,10 +16,8 @@ class FileSystem(Storage):
self._directory = Path(app.instance_path) / config_path
self._files_directory = self._directory / Path("files")
self._meta_directory = self._directory / Path("meta")
self._files_directory.mkdir(mode=0o700, exist_ok=True, parents=True)
self._meta_directory.mkdir(mode=0o700, exist_ok=True, parents=True)
if not self._directory.is_dir():
raise OSError(f"'{app.config['SACHET_FILE_DIR']}' is not a directory.")
@ -28,10 +26,6 @@ class FileSystem(Storage):
name = secure_filename(name)
return self._files_directory / Path(name)
def _get_meta_path(self, name):
name = secure_filename(name)
return self._meta_directory / Path(name)
def list_files(self):
return [
self.get_file(x.name)
@ -47,14 +41,10 @@ class FileSystem(Storage):
self.name = name
self._storage = storage
self._path = self._storage._get_path(name)
self._meta_path = self._storage._get_meta_path(name)
self._path.touch()
self._meta_path.touch()
self.metadata = self._Metadata(self)
def delete(self, name):
self._path.unlink()
self._meta_path.unlink()
def open(self, mode="r"):
return self._path.open(mode=mode)
@ -64,33 +54,4 @@ class FileSystem(Storage):
if new_path.exists():
raise OSError(f"Path {path} already exists.")
new_meta_path = self._storage._get_meta_path(new_name)
if new_meta_path.exists():
raise OSError(f"Path {path} already exists.")
self._path.rename(new_path)
self._meta_path.rename(new_meta_path)
class _Metadata:
def __init__(self, file):
self.__dict__["_file"] = file
@property
def __data(self):
with self._file._meta_path.open() as meta_file:
content = meta_file.read()
if len(content.strip()) == 0:
return {}
return json.loads(content)
# there is no setter for __data because it would cause __setattr__ to infinitely recurse
def __setattr__(self, name, value):
data = self.__data
data[name] = value
with self._file._meta_path.open("w") as meta_file:
content = json.dumps(data)
meta_file.write(content)
def __getattr__(self, name):
return self.__data.get(name, None)

14
tests/conftest.py
View File

@ -1,5 +1,4 @@
import pytest
import yaml
from sachet.server.users import manage
from click.testing import CliRunner
from sachet.server import app, db, storage
@ -7,7 +6,6 @@ from sachet.server.models import Permissions, User
from bitmask import Bitmask
from pathlib import Path
import random
import itertools
@pytest.fixture
@ -23,10 +21,7 @@ def rand():
def clear_filesystem():
if app.config["SACHET_STORAGE"] == "filesystem":
for file in itertools.chain(
storage._meta_directory.iterdir(),
storage._files_directory.iterdir(),
):
for file in storage._files_directory.iterdir():
if file.is_relative_to(Path(app.instance_path)) and file.is_file():
file.unlink()
else:
@ -67,7 +62,12 @@ def users(client):
Returns a dictionary with all the info for each user.
"""
userinfo = dict(
jeff=dict(password="1234", permissions=Bitmask()),
jeff=dict(
password="1234",
permissions=Bitmask(
Permissions.CREATE, Permissions.READ, Permissions.DELETE
),
),
administrator=dict(password="4321", permissions=Bitmask(Permissions.ADMIN)),
)

68
tests/test_files.py Normal file
View File

@ -0,0 +1,68 @@
import pytest
from io import BytesIO
from werkzeug.datastructures import FileStorage
"""Test file share endpoints."""
# if other storage backends are implemented we test them with the same suite
# this might be redundant because test_storage tests the backends already
@pytest.mark.parametrize("client", [{"SACHET_STORAGE": "filesystem"}], indirect=True)
class TestSuite:
def test_sharing(self, client, users, tokens, rand):
# create share
resp = client.post(
"/files", headers={"Authorization": f"bearer {tokens['jeff']}"}
)
assert resp.status_code == 201
data = resp.get_json()
url = data.get("url")
assert url is not None
assert "/files/" in url
upload_data = rand.randbytes(4000)
# upload file to share
resp = client.post(
url + "/content",
headers={"Authorization": f"bearer {tokens['jeff']}"},
data={
"upload": FileStorage(stream=BytesIO(upload_data), filename="upload")
},
content_type="multipart/form-data",
)
assert resp.status_code == 201
# test not allowing re-upload
resp = client.post(
url + "/content",
headers={"Authorization": f"bearer {tokens['jeff']}"},
data={
"upload": FileStorage(stream=BytesIO(upload_data), filename="upload")
},
content_type="multipart/form-data",
)
assert resp.status_code == 423
# read file
resp = client.get(
url + "/content",
headers={"Authorization": f"bearer {tokens['jeff']}"},
)
assert resp.data == upload_data
# test deletion
resp = client.delete(
url,
headers={"Authorization": f"bearer {tokens['jeff']}"},
)
assert resp.status_code == 200
# file shouldn't exist anymore
resp = client.get(
url + "/content",
headers={"Authorization": f"bearer {tokens['jeff']}"},
)
assert resp.status_code == 404

20
tests/test_storage.py
View File

@ -10,18 +10,12 @@ from uuid import UUID
@pytest.mark.parametrize("client", [{"SACHET_STORAGE": "filesystem"}], indirect=True)
class TestSuite:
def test_creation(self, client, rand):
"""Test the process of creating, writing, then reading files with metadata, and also listing files."""
"""Test the process of creating, writing, then reading files, and also listing files."""
files = [
dict(
name=str(UUID(bytes=rand.randbytes(16))),
data=rand.randbytes(4000),
metadata=dict(
sdljkf=dict(abc="def", aaa="bbb"),
lkdsjf=dict(ld="sdlfj", sdljf="sdlkjf"),
sdlfkj="sjdlkfsldk",
ssjdklf=rand.randint(-1000, 1000),
),
)
for i in range(25)
]
@ -30,15 +24,12 @@ class TestSuite:
handle = storage.get_file(file["name"])
with handle.open(mode="wb") as f:
f.write(file["data"])
handle.metadata.test_data = file["metadata"]
for file in files:
handle = storage.get_file(file["name"])
with handle.open(mode="rb") as f:
saved_data = f.read()
assert saved_data == file["data"]
saved_meta = handle.metadata.test_data
assert saved_meta == file["metadata"]
assert sorted([f.name for f in storage.list_files()]) == sorted(
[f["name"] for f in files]
@ -50,12 +41,6 @@ class TestSuite:
name=str(UUID(bytes=rand.randbytes(16))),
new_name=str(UUID(bytes=rand.randbytes(16))),
data=rand.randbytes(4000),
metadata=dict(
sdljkf=dict(abc="def", aaa="bbb"),
lkdsjf=dict(ld="sdlfj", sdljf="sdlkjf"),
sdlfkj="sjdlkfsldk",
ssjdklf=rand.randint(-1000, 1000),
),
)
for i in range(25)
]
@ -64,7 +49,6 @@ class TestSuite:
handle = storage.get_file(file["name"])
with handle.open(mode="wb") as f:
f.write(file["data"])
handle.metadata.test_data = file["metadata"]
handle.rename(file["new_name"])
for file in files:
@ -72,5 +56,3 @@ class TestSuite:
with handle.open(mode="rb") as f:
saved_data = f.read()
assert saved_data == file["data"]
saved_meta = handle.metadata.test_data
assert saved_meta == file["metadata"]