Compare commits

...

7 Commits

9 changed files with 393 additions and 128 deletions

View File

@ -4,8 +4,6 @@
set -gx BASENAME localhost:5000
function sachet_init_db -d "initialize db"
flask --debug --app sachet.server db drop --yes
flask --debug --app sachet.server db create
flask --debug --app sachet.server user create --username admin --admin yes --password password123
flask --debug --app sachet.server user create --username user --admin no --password password123
end
@ -31,13 +29,18 @@ end
function sachet_upload -d "uploads a file"
argparse 's/session=?' -- $argv
set FNAME (basename $argv)
set URL (http --session=$_flag_session post $BASENAME/files file_name=$FNAME | jq -r .url)
http --session=$_flag_session -f post $BASENAME/$URL/content upload@$argv
set URL (http --session=$_flag_session post $BASENAME/files file_name=$FNAME | tee /dev/tty | jq -r .url)
http --session=$_flag_session -f post $BASENAME/$URL/content \
upload@$argv \
dzuuid=(cat /dev/urandom | xxd -ps | head -c 32) \
dzchunkindex=0 \
dztotalchunks=1
end
function sachet_upload_meme -d "uploads a random meme"
argparse 's/session=?' -- $argv
set MEME ~/med/memes/woof/(ls ~/med/memes/woof | shuf | head -n 1)
sachet_upload $MEME
sachet_upload -s$_flag_session $MEME
end
function sachet_list -d "lists files on a given page"

View File

@ -0,0 +1,72 @@
"""empty message
Revision ID: 70ab3c81827a
Revises: 4cd7cdbc2d1f
Create Date: 2023-05-07 21:52:08.250195
"""
from alembic import op
import sqlalchemy as sa
import sqlalchemy_utils
# revision identifiers, used by Alembic.
revision = "70ab3c81827a"
down_revision = "4cd7cdbc2d1f"
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"uploads",
sa.Column("upload_id", sa.String(), nullable=False),
sa.Column("share_id", sqlalchemy_utils.types.uuid.UUIDType(), nullable=True),
sa.Column("create_date", sa.DateTime(), nullable=False),
sa.Column("total_chunks", sa.Integer(), nullable=False),
sa.Column("recv_chunks", sa.Integer(), nullable=False),
sa.Column("completed", sa.Boolean(), nullable=False),
sa.ForeignKeyConstraint(
["share_id"],
["shares.share_id"],
),
sa.PrimaryKeyConstraint("upload_id"),
)
op.create_table(
"chunks",
sa.Column("chunk_id", sa.Integer(), autoincrement=True, nullable=False),
sa.Column("create_date", sa.DateTime(), nullable=False),
sa.Column("index", sa.Integer(), nullable=False),
sa.Column("upload_id", sa.String(), nullable=True),
sa.Column("filename", sa.String(), nullable=False),
sa.ForeignKeyConstraint(
["upload_id"],
["uploads.upload_id"],
),
sa.PrimaryKeyConstraint("chunk_id"),
)
with op.batch_alter_table("shares", schema=None) as batch_op:
batch_op.alter_column(
"share_id",
existing_type=sa.NUMERIC(precision=16),
type_=sqlalchemy_utils.types.uuid.UUIDType(),
existing_nullable=False,
)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table("shares", schema=None) as batch_op:
batch_op.alter_column(
"share_id",
existing_type=sqlalchemy_utils.types.uuid.UUIDType(),
type_=sa.NUMERIC(precision=16),
existing_nullable=False,
)
op.drop_table("chunks")
op.drop_table("uploads")
# ### end Alembic commands ###

View File

@ -21,7 +21,6 @@ class TestingConfig(BaseConfig):
class DevelopmentConfig(BaseConfig):
SERVER_NAME = "localhost.dev"
SQLALCHEMY_DATABASE_URI = sqlalchemy_base + "_dev" + ".db"
BCRYPT_LOG_ROUNDS = 4
SACHET_FILE_DIR = "storage_dev"

View File

@ -2,7 +2,7 @@ import uuid
import io
from flask import Blueprint, request, jsonify, send_file
from flask.views import MethodView
from sachet.server.models import Share, Permissions
from sachet.server.models import Share, Permissions, Upload, Chunk
from sachet.server.views_common import ModelAPI, ModelListAPI, auth_required
from sachet.server import storage, db
@ -68,7 +68,55 @@ files_blueprint.add_url_rule(
)
class FileContentAPI(ModelAPI):
class FileContentAPI(MethodView):
def recv_upload(self, share):
"""Receive chunked uploads.
share : Share
Share we are uploading to.
"""
chunk_file = request.files.get("upload")
if not chunk_file:
return (
jsonify(dict(status="fail", message="Missing chunk data in request.")),
400,
)
chunk_data = chunk_file.read()
try:
dz_uuid = request.form["dzuuid"]
dz_chunk_index = int(request.form["dzchunkindex"])
dz_total_chunks = int(request.form["dztotalchunks"])
except KeyError as err:
return (
jsonify(
dict(status="fail", message=f"Missing data for chunking; {err}")
),
400,
)
except ValueError as err:
return (
jsonify(dict(status="fail", message=f"{err}")),
400,
)
chunk = Chunk(dz_chunk_index, dz_uuid, dz_total_chunks, share, chunk_data)
db.session.add(chunk)
db.session.commit()
upload = chunk.upload
upload.recv_chunks = upload.recv_chunks + 1
if upload.recv_chunks >= upload.total_chunks:
upload.complete()
if upload.completed:
share.initialized = True
db.session.delete(upload)
db.session.commit()
return jsonify(dict(status="success", message="Upload completed.")), 201
else:
return jsonify(dict(status="success", message="Chunk uploaded.")), 200
@auth_required(required_permissions=(Permissions.CREATE,), allow_anonymous=True)
def post(self, share_id, auth_user=None):
share = Share.query.filter_by(share_id=uuid.UUID(share_id)).first()
@ -100,20 +148,7 @@ class FileContentAPI(ModelAPI):
423,
)
upload = request.files["upload"]
data = upload.read()
file = share.get_handle()
with file.open(mode="wb") as f:
f.write(data)
share.initialized = True
db.session.commit()
return (
jsonify({"status": "success", "message": "Share has been initialized."}),
201,
)
return self.recv_upload(share)
@auth_required(required_permissions=(Permissions.MODIFY,), allow_anonymous=True)
def put(self, share_id, auth_user=None):
@ -150,17 +185,7 @@ class FileContentAPI(ModelAPI):
423,
)
upload = request.files["upload"]
data = upload.read()
file = share.get_handle()
with file.open(mode="wb") as f:
f.write(data)
return (
jsonify({"status": "success", "message": "Share has been modified."}),
200,
)
return self.recv_upload(share)
@auth_required(required_permissions=(Permissions.READ,), allow_anonymous=True)
def get(self, share_id, auth_user=None):

View File

@ -6,6 +6,7 @@ from bitmask import Bitmask
from marshmallow import fields, ValidationError
from flask import request, jsonify, url_for, current_app
from sqlalchemy_utils import UUIDType
from sqlalchemy import event
import uuid
@ -281,3 +282,148 @@ class Share(db.Model):
def get_handle(self):
return storage.get_file(str(self.share_id))
@classmethod
def __declare_last__(cls):
@event.listens_for(cls, "before_delete")
def share_before_delete(mapper, connection, share):
file = share.get_handle()
file.delete()
class Upload(db.Model):
"""Upload instance for a given file.
Parameters
----------
upload_id : str
ID associated to this upload.
total_chunks: int
Total amount of chunks in this upload.
share_id : uuid.UUID
Assigns this upload to the given share id.
Attributes
----------
upload_id : str
ID associated to this upload.
total_chunks : int
Total amount of chunks in this upload.
recv_chunks : int
Amount of chunks received in this upload.
completed : bool
Whether the file has been fully uploaded.
share : Share
The share this upload is for.
chunks : list of Chunk
Chunks composing this upload.
create_date : DateTime
Time this upload was started.
"""
__tablename__ = "uploads"
upload_id = db.Column(db.String, primary_key=True)
share_id = db.Column(UUIDType(), db.ForeignKey("shares.share_id"))
share = db.relationship("Share", backref=db.backref("upload"))
create_date = db.Column(db.DateTime, nullable=False)
total_chunks = db.Column(db.Integer, nullable=False)
recv_chunks = db.Column(db.Integer, nullable=False, default=0)
completed = db.Column(db.Boolean, nullable=False, default=False)
chunks = db.relationship(
"Chunk",
backref=db.backref("upload"),
order_by="Chunk.chunk_id",
cascade="all, delete",
)
def __init__(self, upload_id, total_chunks, share_id):
self.share = Share.query.filter_by(share_id=share_id).first()
if self.share is None:
raise KeyError(f"Share '{self.share_id}' could not be found.")
self.upload_id = upload_id
self.total_chunks = total_chunks
self.create_date = datetime.datetime.now()
def complete(self):
"""Merge chunks, save the file, then clean up."""
tmp_file = storage.get_file(f"{self.share.share_id}_{self.upload_id}")
with tmp_file.open(mode="ab") as tmp_f:
for chunk in self.chunks:
chunk_file = storage.get_file(chunk.filename)
with chunk_file.open(mode="rb") as chunk_f:
data = chunk_f.read()
tmp_f.write(data)
# replace the old file
old_file = self.share.get_handle()
old_file.delete()
tmp_file.rename(str(self.share.share_id))
self.completed = True
class Chunk(db.Model):
"""Single chunk within an upload.
Parameters
----------
index : int
Index of this chunk within an upload.
upload_id : str
ID of the upload this chunk is associated to.
total_chunks : int
Total amount of chunks within this upload.
share : Share
Assigns this chunk to the given share.
data : bytes
Raw chunk data.
Attributes
----------
chunk_id : int
ID unique for all chunks (not just in a single upload.)
create_date : DateTime
Time this chunk was received.
index : int
Index of this chunk within an upload.
upload : Upload
Upload this chunk is associated to.
filename : str
Filename the data is stored in.
"""
__tablename__ = "chunks"
chunk_id = db.Column(db.Integer, primary_key=True, autoincrement=True)
create_date = db.Column(db.DateTime, nullable=False)
index = db.Column(db.Integer, nullable=False)
upload_id = db.Column(db.String, db.ForeignKey("uploads.upload_id"))
filename = db.Column(db.String, nullable=False)
def __init__(self, index, upload_id, total_chunks, share, data):
self.upload = Upload.query.filter_by(upload_id=upload_id).first()
if self.upload is None:
self.upload = Upload(upload_id, total_chunks, share.share_id)
self.upload.recv_chunks = 0
db.session.add(self.upload)
self.upload_id = upload_id
self.create_date = datetime.datetime.now()
self.index = index
self.filename = f"{share.share_id}_{self.upload_id}_{self.index}"
file = storage.get_file(self.filename)
with file.open(mode="wb") as f:
f.write(data)
@classmethod
def __declare_last__(cls):
@event.listens_for(cls, "before_delete")
def chunk_before_delete(mapper, connection, chunk):
file = storage.get_file(chunk.filename)
file.delete()

View File

@ -43,7 +43,7 @@ class FileSystem(Storage):
self._path = self._storage._get_path(name)
self._path.touch()
def delete(self, name):
def delete(self):
self._path.unlink()
def open(self, mode="r"):

View File

@ -1,8 +1,12 @@
import pytest
import uuid
from math import ceil
from sachet.server.users import manage
from click.testing import CliRunner
from sachet.server import app, db, storage
from sachet.server.models import Permissions, User
from werkzeug.datastructures import FileStorage
from io import BytesIO
from bitmask import Bitmask
from pathlib import Path
import random
@ -200,3 +204,58 @@ def auth(tokens):
return ret
return auth_headers
@pytest.fixture
def upload(client):
"""Perform chunked upload of some data.
Parameters
----------
url : str
URL to upload to.
data : BytesIO
Stream of data to upload.
You can use BytesIO(data) to convert raw bytes to a stream.
headers : dict, optional
Headers to upload with.
chunk_size : int, optional
Size of chunks in bytes.
method : function
Method like client.post or client.put to use.
"""
def upload(url, data, headers={}, chunk_size=int(2e6), method=client.post):
data_size = len(data.getbuffer())
buf = data.getbuffer()
upload_uuid = uuid.uuid4()
total_chunks = int(ceil(data_size / chunk_size))
resp = None
for chunk_idx in range(total_chunks):
start = chunk_size * chunk_idx
end = min(chunk_size * (chunk_idx + 1), data_size)
resp = method(
url,
headers=headers,
data={
"upload": FileStorage(
stream=BytesIO(buf[start:end]), filename="upload"
),
"dzuuid": str(upload_uuid),
"dzchunkindex": chunk_idx,
"dztotalchunks": total_chunks,
},
content_type="multipart/form-data",
)
if not resp.status_code == 200 or resp.status_code == 201:
break
return resp
return upload

View File

@ -6,7 +6,7 @@ import uuid
"""Test anonymous authentication to endpoints."""
def test_files(client, auth, rand):
def test_files(client, auth, rand, upload):
# set create perm for anon users
resp = client.patch(
"/admin/settings",
@ -28,10 +28,9 @@ def test_files(client, auth, rand):
upload_data = rand.randbytes(4000)
# upload file to share
resp = client.post(
resp = upload(
url + "/content",
data={"upload": FileStorage(stream=BytesIO(upload_data), filename="upload")},
content_type="multipart/form-data",
BytesIO(upload_data),
)
assert resp.status_code == 201
@ -60,12 +59,12 @@ def test_files(client, auth, rand):
# modify share
upload_data = rand.randbytes(4000)
resp = client.put(
resp = upload(
url + "/content",
data={"upload": FileStorage(stream=BytesIO(upload_data), filename="upload")},
content_type="multipart/form-data",
BytesIO(upload_data),
method=client.put,
)
assert resp.status_code == 200
assert resp.status_code == 201
resp = client.patch(
url,
json={"file_name": "new_bin.bin"},
@ -130,7 +129,7 @@ def test_files(client, auth, rand):
assert resp.status_code == 404
def test_files_invalid(client, auth, rand):
def test_files_invalid(client, auth, rand, upload):
# set create perm for anon users
resp = client.patch(
"/admin/settings",
@ -151,11 +150,7 @@ def test_files_invalid(client, auth, rand):
data = resp.get_json()
url = data.get("url")
upload_data = rand.randbytes(4000)
resp = client.post(
url + "/content",
data={"upload": FileStorage(stream=BytesIO(upload_data), filename="upload")},
content_type="multipart/form-data",
)
resp = upload(url + "/content", BytesIO(upload_data))
assert resp.status_code == 201
# disable all permissions
@ -167,28 +162,15 @@ def test_files_invalid(client, auth, rand):
assert resp.status_code == 200
# test initializing a share without perms
resp = client.post(
uninit_url + "/content",
data={"upload": FileStorage(stream=BytesIO(upload_data), filename="upload")},
content_type="multipart/form-data",
)
resp = upload(url + "/content", BytesIO(upload_data))
assert resp.status_code == 401
# test reading a share without perms
resp = client.get(url + "/content")
# test modifying an uninitialized share without perms
resp = client.put(
uninit_url + "/content",
data={"upload": FileStorage(stream=BytesIO(upload_data), filename="upload")},
content_type="multipart/form-data",
)
assert resp.status_code == 401
resp = upload(uninit_url + "/content", BytesIO(upload_data), method=client.put)
assert resp.status_code == 401
# test modifying a share without perms
resp = client.put(
url + "/content",
data={"upload": FileStorage(stream=BytesIO(upload_data), filename="upload")},
content_type="multipart/form-data",
)
resp = upload(url + "/content", BytesIO(upload_data), method=client.put)
assert resp.status_code == 401
# test deleting a share without perms

View File

@ -1,6 +1,8 @@
import pytest
from os.path import basename
from io import BytesIO
from werkzeug.datastructures import FileStorage
from sachet.server import storage
import uuid
"""Test file share endpoints."""
@ -10,7 +12,7 @@ import uuid
# this might be redundant because test_storage tests the backends already
@pytest.mark.parametrize("client", [{"SACHET_STORAGE": "filesystem"}], indirect=True)
class TestSuite:
def test_sharing(self, client, users, auth, rand):
def test_sharing(self, client, users, auth, rand, upload):
# create share
resp = client.post(
"/files", headers=auth("jeff"), json={"file_name": "content.bin"}
@ -25,14 +27,11 @@ class TestSuite:
upload_data = rand.randbytes(4000)
# upload file to share
resp = client.post(
resp = upload(
url + "/content",
BytesIO(upload_data),
headers=auth("jeff"),
data={
"upload": FileStorage(stream=BytesIO(upload_data), filename="upload")
},
content_type="multipart/form-data",
chunk_size=1230,
)
assert resp.status_code == 201
@ -58,7 +57,10 @@ class TestSuite:
)
assert resp.status_code == 404
def test_modification(self, client, users, auth, rand):
for f in storage.list_files():
assert basename(url) not in f.name
def test_modification(self, client, users, auth, rand, upload):
# create share
resp = client.post(
"/files", headers=auth("jeff"), json={"file_name": "content.bin"}
@ -70,14 +72,12 @@ class TestSuite:
new_data = rand.randbytes(4000)
# upload file to share
resp = client.post(
resp = upload(
url + "/content",
BytesIO(upload_data),
headers=auth("jeff"),
data={
"upload": FileStorage(stream=BytesIO(upload_data), filename="upload")
},
content_type="multipart/form-data",
)
assert resp.status_code == 201
# modify metadata
resp = client.patch(
@ -88,12 +88,13 @@ class TestSuite:
assert resp.status_code == 200
# modify file contents
resp = client.put(
resp = upload(
url + "/content",
BytesIO(new_data),
headers=auth("jeff"),
data={"upload": FileStorage(stream=BytesIO(new_data), filename="upload")},
method=client.put,
)
assert resp.status_code == 200
assert resp.status_code == 201
# read file
resp = client.get(
@ -103,7 +104,7 @@ class TestSuite:
assert resp.data == new_data
assert "filename=new_bin.bin" in resp.headers["Content-Disposition"].split("; ")
def test_invalid(self, client, users, auth, rand):
def test_invalid(self, client, users, auth, rand, upload):
"""Test invalid requests."""
upload_data = rand.randbytes(4000)
@ -141,66 +142,51 @@ class TestSuite:
url = data.get("url")
# test invalid methods
resp = client.put(
resp = upload(
url + "/content",
BytesIO(upload_data),
headers=auth("jeff"),
data={
"upload": FileStorage(stream=BytesIO(upload_data), filename="upload")
},
content_type="multipart/form-data",
method=client.put,
)
assert resp.status_code == 423
resp = client.patch(
resp = upload(
url + "/content",
BytesIO(upload_data),
headers=auth("jeff"),
data={
"upload": FileStorage(stream=BytesIO(upload_data), filename="upload")
},
content_type="multipart/form-data",
method=client.patch,
)
assert resp.status_code == 405
# test other user being unable to upload to this share
resp = client.post(
resp = upload(
url + "/content",
BytesIO(upload_data),
headers=auth("dave"),
data={
"upload": FileStorage(stream=BytesIO(upload_data), filename="upload")
},
content_type="multipart/form-data",
)
assert resp.status_code == 403
# upload file to share (properly)
resp = client.post(
resp = upload(
url + "/content",
BytesIO(upload_data),
headers=auth("jeff"),
data={
"upload": FileStorage(stream=BytesIO(upload_data), filename="upload")
},
content_type="multipart/form-data",
)
assert resp.status_code == 201
# test other user being unable to modify this share
resp = client.put(
resp = upload(
url + "/content",
BytesIO(upload_data),
headers=auth("dave"),
data={
"upload": FileStorage(stream=BytesIO(upload_data), filename="upload")
},
content_type="multipart/form-data",
method=client.put,
)
assert resp.status_code == 403
# test not allowing re-upload
resp = client.post(
resp = upload(
url + "/content",
BytesIO(upload_data),
headers=auth("jeff"),
data={
"upload": FileStorage(stream=BytesIO(upload_data), filename="upload")
},
content_type="multipart/form-data",
)
assert resp.status_code == 423
@ -210,7 +196,7 @@ class TestSuite:
resp = client.get(url + "/content", headers=auth("no_read_user"))
assert resp.status_code == 403
def test_locking(self, client, users, auth, rand):
def test_locking(self, client, users, auth, rand, upload):
# upload share
resp = client.post(
"/files", headers=auth("jeff"), json={"file_name": "content.bin"}
@ -218,13 +204,10 @@ class TestSuite:
data = resp.get_json()
url = data.get("url")
upload_data = rand.randbytes(4000)
resp = client.post(
resp = upload(
url + "/content",
BytesIO(upload_data),
headers=auth("jeff"),
data={
"upload": FileStorage(stream=BytesIO(upload_data), filename="upload")
},
content_type="multipart/form-data",
)
assert resp.status_code == 201
@ -236,13 +219,11 @@ class TestSuite:
assert resp.status_code == 200
# attempt to modify share
resp = client.put(
resp = upload(
url + "/content",
BytesIO(upload_data),
headers=auth("jeff"),
data={
"upload": FileStorage(stream=BytesIO(upload_data), filename="upload")
},
content_type="multipart/form-data",
method=client.put,
)
assert resp.status_code == 423
@ -261,15 +242,13 @@ class TestSuite:
assert resp.status_code == 200
# attempt to modify share
resp = client.put(
resp = upload(
url + "/content",
BytesIO(upload_data),
headers=auth("jeff"),
data={
"upload": FileStorage(stream=BytesIO(upload_data), filename="upload")
},
content_type="multipart/form-data",
method=client.put,
)
assert resp.status_code == 200
assert resp.status_code == 201
# attempt to delete share
resp = client.delete(