[Date Prev][Date Next] [Thread Prev][Thread Next] [Date Index] [Thread Index]

[Git][snapshot-team/snapshot][master] 9 commits: Add support for optional filters on queries with SnapshotModel._modular_query()

Title: GitLab

Baptiste Beauplat pushed to branch master at snapshot / snapshot

Commits:

  • e376760b
    by Baptiste Beauplat at 2024-07-15T21:35:52+02:00 
    Add support for optional filters on queries with SnapshotModel._modular_query()
  • 6819e8de
    by Baptiste Beauplat at 2024-07-15T21:35:52+02:00 
    Add mirrorruns_get_mirrorrun() method to fetch run and archive from db
  • b4224d48
    by Baptiste Beauplat at 2024-07-15T21:35:52+02:00 
    Add /mr/timestamp/ route to serve a list of snapshot timestamp (Closes: #969603)

Can be optionally filtered using the query_parameters archive, since and
before.
  • 22362291
    by Baptiste Beauplat at 2024-07-15T21:35:52+02:00 
    Document new /mr/timestamp API endpoint
  • bcb368e2
    by Baptiste Beauplat at 2024-07-15T21:35:52+02:00 
    Add date fixture to test to parse string timestamp into datetime
  • 7263528a
    by Baptiste Beauplat at 2024-07-15T21:35:52+02:00 
    Add get_runs() method to testing dataset to retrieve a list of runs and archives
  • 8b1c1351
    by Baptiste Beauplat at 2024-07-15T21:35:52+02:00 
    Add new unit test to cover /mr/timestamp endpoint
  • be8b5ed3
    by Baptiste Beauplat at 2024-07-15T21:35:52+02:00 
    Add a new archive to testing dataset to cover a missing branch in _index_query_results()
  • 85f83e88
    by Baptiste Beauplat at 2024-07-25T20:31:03+02:00 
    Merge branch 'feat/timestamp-endpoint'

8 changed files:

Changes:

  • API
    ... ... @@ -152,10 +152,19 @@ note: different source packages can build binaries with the same binary package
    152 152 
             ]
    153 153 
     }
    154 154
    155 
    -
    156 
    -
    157 
    -
    158 
    -
    155 
    +URL: /mr/timestamp
    156 
    +Options: after=date  get timestamp after specified date
    157 
    + before=date         get timestamp before specified date
    158 
    + archive=name        get timestamp for this archive only
    159 
    +http status codes: 200 500
    160 
    +summary: list all timestamp of snapshot runs, indexed by archive
    161 
    +{   "_comment": "yadayadayda",
    162 
    +    "result":
    163 
    +        {   "debian": ["20230103T151722Z", "20230105T152319Z"],
    164 
    +            "debian-ports": ["20230104T151406Z"],
    165 
    +            ...
    166 
    +        }
    167 
    +}
    159 168
    160 169 
     URL: /file/<hash>
    161 170 
     http status codes: 200 500 403 404 451 304

  • web/app/snapshot/models/snapshot.py
    ... ... @@ -134,6 +134,43 @@ class SnapshotModel:
    134 134
    135 135 
             return result
    136 136
    137 
    +    def mirrorruns_get_mirrorrun(self, archive, after, before):
    138 
    +        head = """
    139 
    +            SELECT run, name as archive
    140 
    +              FROM mirrorrun JOIN archive
    141 
    +                USING (archive_id)
    142 
    +        """
    143 
    +        tail = 'ORDER BY run'
    144 
    +        filters = {
    145 
    +            'archive': 'archive.name = %(archive)s',
    146 
    +            'after': 'run >= %(after)s',
    147 
    +            'before': 'run <= %(before)s',
    148 
    +        }
    149 
    +
    150 
    +        return self._modular_query(head, filters, tail, archive=archive,
    151 
    +                                   after=after, before=before)
    152 
    +
    153 
    +    def _modular_query(self, head, filters, tail, **kwargs):
    154 
    +        condition_keyword = 'WHERE'
    155 
    +        condition = ''
    156 
    +
    157 
    +        for key, value in list(kwargs.items()):
    158 
    +            if value is None:
    159 
    +                kwargs.pop(key)
    160 
    +                continue
    161 
    +
    162 
    +            condition += f'{condition_keyword} {filters[key]}'
    163 
    +
    164 
    +            if condition_keyword == 'WHERE':
    165 
    +                condition_keyword = ' AND'
    166 
    +
    167 
    +        query = f'{head} {condition} {tail}'
    168 
    +
    169 
    +        with DBInstance(self.pool) as db:
    170 
    +            rows = db.query(query, kwargs)
    171 
    +
    172 
    +        return rows
    173 
    +
    137 174 
         # @beaker_cache(expire=600, cache_response=False, type='memory',
    138 175 
         #               key='archive')
    139 176 
         # def mirrorruns_get_etag(self, archive):

  • web/app/snapshot/settings/common.py
    ... ... @@ -41,6 +41,7 @@ CACHE_TIMEOUT_DEFAULT = 600
    41 41 
     CACHE_TIMEOUT_MR_INDEX = CACHE_TIMEOUT_DEFAULT
    42 42 
     CACHE_TIMEOUT_MR_PACKAGE = CACHE_TIMEOUT_DEFAULT
    43 43 
     CACHE_TIMEOUT_MR_VERSION = CACHE_TIMEOUT_DEFAULT
    44 
    +CACHE_TIMEOUT_MR_TIMESTAMP = CACHE_TIMEOUT_DEFAULT
    44 45
    45 46 
     CACHE_TIMEOUT_REMOVAL = CACHE_TIMEOUT_DEFAULT
    46 47 
     CACHE_TIMEOUT_REMOVAL_ONE = CACHE_TIMEOUT_DEFAULT

  • web/app/snapshot/views/mr.py
    1 1 
     # snapshot.debian.org - web frontend
    2 2 
     #
    3 3 
     # Copyright (c) 2009, 2010, 2015 Peter Palfrader
    4 
    -# Copyright (c) 2021 Baptiste Beauplat <lyknode@cilg.org>
    4 
    +# Copyright (c) 2021, 2023 Baptiste Beauplat <lyknode@cilg.org>
    5 5 
     #
    6 6 
     # Permission is hereby granted, free of charge, to any person obtaining a copy
    7 7 
     # of this software and associated documentation files (the "Software"), to deal
    ... ... @@ -54,6 +54,29 @@ def _get_fileinfo_for_mr(hashes):
    54 54 
         return result
    55 55
    56 56
    57 
    +@router.route("/timestamp/")
    58 
    +@cache()
    59 
    +def mr_timestamp():
    60 
    +    mr_timestamp.cache_timeout = current_app.config[
    61 
    +        'CACHE_TIMEOUT_MR_TIMESTAMP'
    62 
    +    ]
    63 
    +    after = request.args.get('after')
    64 
    +    before = request.args.get('before')
    65 
    +    archive = request.args.get('archive')
    66 
    +    timestamps = {}
    67 
    +
    68 
    +    runs = get_snapshot_model().mirrorruns_get_mirrorrun(archive, after,
    69 
    +                                                         before)
    70 
    +
    71 
    +    for timestamp, archive in runs:
    72 
    +        timestamps.setdefault(archive, []).append(rfc3339_timestamp(timestamp))
    73 
    +
    74 
    +    return jsonify({
    75 
    +        '_comment': "foo",
    76 
    +        'result': timestamps,
    77 
    +    })
    78 
    +
    79 
    +
    57 80 
     @router.route("/package/")
    58 81 
     @cache()
    59 82 
     def mr_index():

  • web/app/tests/conftest.py
    ... ... @@ -24,6 +24,7 @@
    24 24 
     from os.path import realpath, abspath, join
    25 25 
     from sys import path
    26 26 
     from logging import getLogger
    27 
    +from datetime import datetime
    27 28
    28 29 
     from testing.postgresql import PostgresqlFactory
    29 30 
     from pytest import fixture
    ... ... @@ -130,6 +131,17 @@ def client(app):
    130 131 
         return app.test_client()
    131 132
    132 133
    134 
    +@fixture
    135 
    +def date():
    136 
    +    def _parse(data):
    137 
    +        if ':' in data:
    138 
    +            return datetime.strptime(data, '%Y-%m-%dT%H:%M:%SZ')
    139 
    +        else:
    140 
    +            return datetime.strptime(data, '%Y%m%dT%H%M%SZ')
    141 
    +
    142 
    +    return _parse
    143 
    +
    144 
    +
    133 145 
     @fixture
    134 146 
     def snapshot():
    135 147 
         return Snapshot

  • web/app/tests/controllers/dataset.py
    ... ... @@ -73,3 +73,12 @@ class DatasetController():
    73 73 
                                 versions.add(version)
    74 74
    75 75 
             return sorted(versions)
    76 
    +
    77 
    +    def get_runs(self):
    78 
    +        runs = {}
    79 
    +
    80 
    +        for run, archives in self.dataset.items():
    81 
    +            for archive in archives.keys():
    82 
    +                runs.setdefault(archive, []).append(run)
    83 
    +
    84 
    +        return runs

  • web/app/tests/data/dataset.py
    ... ... @@ -41,6 +41,11 @@ SNAPSHOT_DATASET = {
    41 41 
             },
    42 42 
         },
    43 43 
         '2020-08-11T18:54:38Z': {
    44 
    +        'debian-ports': {
    45 
    +            'unstable': {
    46 
    +                'zsh': '1.10',
    47 
    +            },
    48 
    +        },
    44 49 
             'debian': {
    45 50 
                 'unstable': {
    46 51 
                     'htop': '1.42',

  • web/app/tests/unit/views/test_mr.py
    1 1 
     # snapshot.debian.org - web frontend
    2 2 
     # https://salsa.debian.org/snapshot-team/snapshot
    3 3 
     #
    4 
    -# Copyright (c) 2021 Baptiste Beauplat <lyknode@cilg.org>
    4 
    +# Copyright (c) 2021, 2023 Baptiste Beauplat <lyknode@cilg.org>
    5 5 
     #
    6 6 
     # Permission is hereby granted, free of charge, to any person obtaining a copy
    7 7 
     # of this software and associated documentation files (the "Software"), to deal
    ... ... @@ -42,6 +42,56 @@ def test_mr_index(client, snapshot_db):
    42 42 
         assert data == expected
    43 43
    44 44
    45 
    +@mark.parametrize('archive,after,before', (
    46 
    +    (False, False, False),
    47 
    +    (False, True, False),
    48 
    +    (False, False, True),
    49 
    +    (False, True, True),
    50 
    +    (True, True, True),
    51 
    +))
    52 
    +def test_mr_timestamp(client, archive, after, before, dataset, date):
    53 
    +    expected = dataset.get_runs()
    54 
    +    after_time = '2020-08-11T00:00:00Z'
    55 
    +    before_time = '2020-08-12T00:00:00Z'
    56 
    +    archive_name = 'debian-ports'
    57 
    +    query_string = {}
    58 
    +
    59 
    +    if after:
    60 
    +        query_string['after'] = after_time
    61 
    +
    62 
    +    if before:
    63 
    +        query_string['before'] = before_time
    64 
    +
    65 
    +    if archive:
    66 
    +        query_string['archive'] = archive_name
    67 
    +
    68 
    +    for key in list(expected.keys()):
    69 
    +        if archive and archive_name != key:
    70 
    +            expected.pop(key)
    71 
    +            continue
    72 
    +
    73 
    +        expected[key].sort()
    74 
    +        expected[key] = list(map(lambda x: x.replace('-', '').replace(':', ''),
    75 
    +                                 expected[key]))
    76 
    +
    77 
    +        for run in list(expected[key]):
    78 
    +            if after and date(run) < date(after_time):
    79 
    +                expected[key].remove(run)
    80 
    +                continue
    81 
    +
    82 
    +            if before and date(run) > date(before_time):
    83 
    +                expected[key].remove(run)
    84 
    +                continue
    85 
    +
    86 
    +    response = client.get('/mr/timestamp/', query_string=query_string)
    87 
    +    data = loads(response.data.decode())
    88 
    +
    89 
    +    assert data == {
    90 
    +        '_comment': 'foo',
    91 
    +        'result': expected
    92 
    +    }
    93 
    +
    94 
    +
    45 95 
     @mark.parametrize('package,status', (
    46 96 
         ('zsh', 200,),
    47 97 
         ('this-package-does-not-exists', 404,),

    • —
    View it on GitLab.
    You're receiving this email because of your account on salsa.debian.org. Manage all notifications · Help

    Reply to: