diff --git a/.coveragerc b/.coveragerc index 6759c46..26e5bf1 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,4 +1,5 @@ [report] omit = + runbench.py runtests.py setup.py diff --git a/.gitignore b/.gitignore index b083645..d366ff9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +/mysite +/manage.py + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -58,4 +61,4 @@ docs/_build/ # Vim *~ *.swp -*.swo \ No newline at end of file +*.swo diff --git a/CHANGES.md b/CHANGES.md index 3fface6..8deefd7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,8 @@ +1.2.0 +================== + +* Support paginating by multiple fields + 1.1.0 ================== diff --git a/Makefile b/Makefile index 87093ce..93ba7a7 100644 --- a/Makefile +++ b/Makefile @@ -13,4 +13,16 @@ sdist: test clean release: sdist twine upload dist/* +bench_build: + cd bench && docker-compose build + +bench_clean: + cd bench \ + && docker-compose stop \ + && docker-compose rm --force -v + +bench_run: + cd bench \ + && docker-compose run --rm --entrypoint '/bin/sh -c' paginator '/bin/sh' + .PHONY: clean docs test sdist release diff --git a/README.md b/README.md index a4b8370..a15dae7 100644 --- a/README.md +++ b/README.md @@ -22,13 +22,13 @@ based on the top/last keyset This approach has two main advantages over the *OFFSET/LIMIT* approach: * is correct: unlike the *offset/limit* based approach it correctly handles -new entries and deleted entries. Last row of Page 4 does not show up as first -row of Page 5 just because row 23 on Page 2 was deleted in the meantime. -Nor do rows mysteriously vanish between pages. These anomalies are common -with the *offset/limit* based approach, but the *keyset* based solution does -a much better job at avoiding them. + new entries and deleted entries. Last row of Page 4 does not show up as first + row of Page 5 just because row 23 on Page 2 was deleted in the meantime. + Nor do rows mysteriously vanish between pages. These anomalies are common + with the *offset/limit* based approach, but the *keyset* based solution does + a much better job at avoiding them. * is fast: all operations can be solved with a fast row positioning followed -by a range scan in the desired direction. + by a range scan in the desired direction. For a full explanation go to [the seek method](http://use-the-index-luke.com/sql/partial-results/fetch-next-page) @@ -46,11 +46,6 @@ infinite-scroll-pagination requires the following software to be installed: pip install django-infinite-scroll-pagination ``` -## Django Rest Framework (DRF) - -DRF has the built-in `CursorPagination` -that is similar to this lib. Use that instead. - ## Usage This example paginates by a `created_at` date field: @@ -102,12 +97,24 @@ def pagination_ajax(request): return HttpResponse(json.dumps(data), content_type="application/json") ``` -Paginating by pk, id or some `unique=True` field: +Paginating by `pk`, `id`, or some `unique=True` field: ```python page = paginator.paginate(queryset, lookup_field='pk', value=pk, per_page=20) ``` +Paginating by multiple fields: + +```python +page = paginator.paginate( + queryset, + lookup_field=('-is_pinned', '-created_at', '-pk'), + value=(is_pinned, created_at, pk), + per_page=20) +``` + +> Make sure the last field is `unique=True`, or `pk` + ## Items order DESC order: @@ -178,14 +185,23 @@ class Article(models.Model): class Meta: indexes = [ - models.Index(fields=['created_at', 'pk'], - models.Index(fields=['-created_at', '-pk'])] + models.Index(fields=['created_at', 'id']), + models.Index(fields=['-created_at', '-id'])] ``` > Note: an index is require for both directions, since the query has a `LIMIT`. See [indexes-ordering](https://www.postgresql.org/docs/9.3/indexes-ordering.html) +However, this library does not implements the fast "row values" +variant of [the seek method](https://use-the-index-luke.com/sql/partial-results/fetch-next-page). +What this means is the index is only +used on the first field. If the first field is a boolean, +then it won't be used. So, it's pointless to index anything other than the first field. +See [PR #8](https://github.com/nitely/django-infinite-scroll-pagination/pull/8) +if you are interested in benchmarks numbers, and please let me know +if there is a way to implement the "row values" variant without using raw SQL. + Pass a limit to the following methods, or use them in places where there won't be many records, otherwise they get expensive fast: diff --git a/bench/Dockerfile b/bench/Dockerfile new file mode 100644 index 0000000..d8db950 --- /dev/null +++ b/bench/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.7.2-alpine + +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 + +RUN apk update \ + && apk add \ + postgresql-client \ + postgresql-dev \ + gcc \ + musl-dev \ + make \ + libffi-dev + +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app + +RUN pip install --upgrade pip \ + && pip install Django==2.2.8 \ + && pip install psycopg2-binary==2.8.6 + +CMD until pg_isready --username=postgres --host=database; do sleep 1; done; +ENTRYPOINT /bin/sh diff --git a/bench/docker-compose.yml b/bench/docker-compose.yml new file mode 100644 index 0000000..5857230 --- /dev/null +++ b/bench/docker-compose.yml @@ -0,0 +1,13 @@ +version: '3' + +services: + database: + image: postgres:10.5 + restart: always + paginator: + build: . + hostname: paginator + volumes: + - ..:/usr/src/app + links: + - database diff --git a/infinite_scroll_pagination/paginator.py b/infinite_scroll_pagination/paginator.py index ce3717c..25d1f07 100644 --- a/infinite_scroll_pagination/paginator.py +++ b/infinite_scroll_pagination/paginator.py @@ -1,13 +1,12 @@ #-*- coding: utf-8 -*- - try: from collections.abc import Sequence except ImportError: from collections import Sequence from django.core.paginator import EmptyPage -from django.db.models import QuerySet +from django.db.models import QuerySet, Q __all__ = [ 'SeekPaginator', @@ -17,8 +16,7 @@ 'PREV_PAGE'] -NEXT_PAGE = 1 -PREV_PAGE = 2 +NEXT_PAGE, PREV_PAGE, DESC, ASC = range(1, 5) class _NoPk: @@ -33,52 +31,73 @@ def __repr__(self): # the first page _NO_PK = _NoPk() +# XXX simplify things by removing the pk parameter, +# and requiring it as last field/value; we should +# also validate there is a single unique=True field, +# and it's the last one; this is a breaking chance, though -class SeekPaginator(object): +class SeekPaginator: def __init__(self, query_set, per_page, lookup_field): assert isinstance(query_set, QuerySet), 'QuerySet expected' assert isinstance(per_page, int), 'Int expected' - assert isinstance(lookup_field, str), 'String expected' + #assert isinstance(lookup_field, str), 'String expected' self.query_set = query_set self.per_page = per_page - self.is_desc = lookup_field.startswith('-') - self.is_asc = not self.is_desc - self.lookup_field = lookup_field.lstrip('-') + if isinstance(lookup_field, str): + lookup_field = (lookup_field,) + self.lookup_fields = lookup_field + + @property + def fields(self): + return tuple(f.lstrip('-') for f in self.lookup_fields) + + @property + def fields_direction(self): + d = {True: DESC, False: ASC} + return tuple( + (f.lstrip('-'), d[f.startswith('-')]) + for f in self.lookup_fields) def prepare_order(self, has_pk, move_to): - pk_sort = 'pk' - lookup_sort = self.lookup_field - if ((self.is_desc and move_to == NEXT_PAGE) or - (self.is_asc and move_to == PREV_PAGE)): - pk_sort = '-%s' % pk_sort - lookup_sort = '-%s' % lookup_sort + fields = list(self.fields_direction) if has_pk: - return [lookup_sort, pk_sort] - return [lookup_sort] - - def prepare_lookup(self, value, pk, move_to): - lookup_include = '%s__gt' % self.lookup_field - lookup_exclude_pk = 'pk__lte' - if ((self.is_desc and move_to == NEXT_PAGE) or - (self.is_asc and move_to == PREV_PAGE)): - lookup_include = '%s__lt' % self.lookup_field - lookup_exclude_pk = 'pk__gte' - lookup_exclude = None - if pk is not _NO_PK: - lookup_include = "%se" % lookup_include - lookup_exclude = {self.lookup_field: value, lookup_exclude_pk: pk} - lookup_filter = {lookup_include: value} - return lookup_filter, lookup_exclude + fields.append( + ('pk', fields[-1][1])) + result = [] + for f, d in fields: + if ((d == DESC and move_to == NEXT_PAGE) or + (d == ASC and move_to == PREV_PAGE)): + f = '-%s' % f + result.append(f) + return result + + # q = X<=? & ~(X=? & ~(Y= ?) ORDER BY date DESC, id DESC + Multi field lookup. Note how it produces nesting, + and how I removed it using boolean logic simplification:: + + X <= ? + AND NOT (X = ? AND (date <= ? AND NOT (date = ? AND id >= ?))) + <---> + X <= ? + AND (NOT X = ? OR NOT date <= ? OR (date = ? AND id >= ?)) + <---> + X <= ? + AND (NOT X = ? OR NOT date <= ? OR date = ?) + AND (NOT X = ? OR NOT date <= ? OR id >= ?) + + A * ~(B * (C * ~(D * F))) + -> (D + ~B + ~C) * (F + ~B + ~C) * A + A * ~(B * (C * ~(D * (F * ~(G * H))))) + -> (D + ~B + ~C) * (F + ~B + ~C) * (~B + ~C + ~G + ~H) * A + A * ~(B * (C * ~(D * (F * ~(G * (X * ~(Y * Z))))))) + -> (D + ~B + ~C) * (F + ~B + ~C) * (Y + ~B + ~C + ~G + ~X) * (Z + ~B + ~C + ~G + ~X) * A + + Addendum:: + + X <= ? + AND NOT (X = ? AND NOT (date <= ? AND NOT (date = ? AND id >= ?))) + """ query_set = self.query_set if value is not None: + if not isinstance(value, (tuple, list)): + value = (value,) query_set = self.apply_filter( value=value, pk=pk, move_to=move_to) query_set = query_set.order_by( @@ -125,7 +171,6 @@ def page(self, value, pk=_NO_PK, move_to=NEXT_PAGE): class SeekPage(Sequence): - def __init__(self, query_set, key, move_to, paginator): self._query_set = query_set self._key = key @@ -163,8 +208,11 @@ def _some_seek(self, direction): pk = _NO_PK if self._key['pk'] is not _NO_PK: pk = last.pk + values = tuple( + getattr(last, f) + for f in self.paginator.fields) return self.paginator.seek( - value=getattr(last, self.paginator.lookup_field), + value=values, pk=pk, move_to=direction) @@ -214,10 +262,10 @@ def prev_pages_left(self, limit=None): def _some_page(self, index): if not self.object_list: return {} - key = { - 'value': getattr( - self.object_list[index], - self.paginator.lookup_field)} + values = tuple( + getattr(self.object_list[index], f) + for f in self.paginator.fields) + key = {'value': values} if self._key['pk'] is not _NO_PK: key['pk'] = self.object_list[index].pk return key diff --git a/infinite_scroll_pagination/serializers.py b/infinite_scroll_pagination/serializers.py index b9a9a6f..85c8953 100644 --- a/infinite_scroll_pagination/serializers.py +++ b/infinite_scroll_pagination/serializers.py @@ -63,6 +63,8 @@ def to_page_key(value=None, pk=None): """Serialize a value and pk to `timestamp-pk`` format""" if value is None: return '' + if isinstance(value, (tuple, list)): + (value,) = value value = _make_aware_maybe(value) try: timestamp = value.timestamp() diff --git a/runbench.py b/runbench.py new file mode 100644 index 0000000..f1ac450 --- /dev/null +++ b/runbench.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import datetime + +import django +from django.conf import settings +from django.utils import timezone +from django.core.management import call_command + + +def django_setup(): + settings.configure( + DATABASES={ + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': 'postgres', + 'USER': 'postgres', + 'PASSWORD': 'postgres', + 'HOST': 'database', + 'PORT': '5432', + } + }, + INSTALLED_APPS=[ + "tests", + ], + ROOT_URLCONF="tests.urls", + DEBUG=False, + ) + django.setup() + call_command('migrate') + +#from tests.models import Article +#from infinite_scroll_pagination.paginator import SeekPaginator +#from infinite_scroll_pagination import paginator as inf_paginator + +def populate_db(): + from tests.models import Article + if Article.objects.all().count() > 0: + print('Some records found; skipping') + return + Article.objects.all().delete() + date = timezone.now() + # change to range(5) to create 5M records + for n in range(1): + articles = [] + for i in range(1_000_000): + seconds = datetime.timedelta(microseconds=i+1_000_000*n) + articles.append(Article( + title="%s" % (i+1_000_000*n), date=date, date_unique=date + seconds)) + Article.objects.bulk_create(articles) + + +def bench(): + from timeit import default_timer as timer + from django.core.paginator import Paginator + from tests.models import Article + from infinite_scroll_pagination.paginator import SeekPaginator + articles = Article.objects.all().order_by( + '-is_pinned', '-is_sticky', "-date", '-pk') + #'-is_pinned', '-is_sticky', + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_pinned = True + a.is_sticky = True + a.save() + for a in articles[810_000:810_050]: + a.is_pinned = True + a.save() + for a in articles[510_000:510_050]: + a.is_pinned = True + a.save() + for a in articles[910_000:910_050]: + a.is_sticky = True + a.save() + + start = timer() + article1 = list(articles[800_000:800_010])[0] + #article1 = list(articles[4_000_000:4_000_010])[0] + end = timer() + print("Offset/Limit", end - start) + + start = timer() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', '-is_sticky', '-date',)) + page = paginator.page( + value=( + article1.is_pinned, + article1.is_sticky, + article1.date,), + pk=article1.pk) + assert list(page) + end = timer() + print("Seek Method", end - start) + + +def start(): + django_setup() + print('Populating DB') + populate_db() + print('Running bench') + bench() + + +if __name__ == "__main__": + start() diff --git a/runtests.py b/runtests.py index f91505e..e1db62e 100644 --- a/runtests.py +++ b/runtests.py @@ -1,9 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import unicode_literals - -import os import sys import logging diff --git a/tests/migrations/0002_article_is_pinned.py b/tests/migrations/0002_article_is_pinned.py new file mode 100644 index 0000000..236b177 --- /dev/null +++ b/tests/migrations/0002_article_is_pinned.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.1 on 2020-10-06 07:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tests', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='article', + name='is_pinned', + field=models.BooleanField(default=False), + ), + ] diff --git a/tests/migrations/0003_article_is_sticky.py b/tests/migrations/0003_article_is_sticky.py new file mode 100644 index 0000000..d12b667 --- /dev/null +++ b/tests/migrations/0003_article_is_sticky.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.1 on 2020-10-07 04:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tests', '0002_article_is_pinned'), + ] + + operations = [ + migrations.AddField( + model_name='article', + name='is_sticky', + field=models.BooleanField(default=False), + ), + ] diff --git a/tests/models.py b/tests/models.py index c64b4b4..923b7d0 100644 --- a/tests/models.py +++ b/tests/models.py @@ -10,6 +10,18 @@ class Article(models.Model): title = models.CharField(max_length=75) date = models.DateTimeField() date_unique = models.DateTimeField(unique=True) + is_pinned = models.BooleanField(default=False) + is_sticky = models.BooleanField(default=False) + + #class Meta: + # The benchmarks show index are not used, see + # https://github.com/nitely/django-infinite-scroll-pagination/pull/8 + #indexes = [ + # models.Index(fields=['is_pinned', 'is_sticky', 'date', 'id']), + # models.Index(fields=['-is_pinned', '-is_sticky', '-date', '-id'])] + #indexes = [ + # models.Index(fields=['date', 'id']), + # models.Index(fields=['-date', '-id'])] def __unicode__(self): - return self.title \ No newline at end of file + return self.title diff --git a/tests/test_multi_fields.py b/tests/test_multi_fields.py new file mode 100644 index 0000000..3e35513 --- /dev/null +++ b/tests/test_multi_fields.py @@ -0,0 +1,838 @@ +#-*- coding: utf-8 -*- + +import datetime + +from django.test import TestCase +from django.utils import timezone + +from .models import Article +from infinite_scroll_pagination.paginator import SeekPaginator +from infinite_scroll_pagination import paginator as inf_paginator + + +class Paginator2FieldsTest(TestCase): + + def setUp(self): + date = timezone.now() + for i in range(25): + seconds = datetime.timedelta(seconds=i) + Article.objects.create( + title="%s" % i, date=date, date_unique=date + seconds) + + def test_next_desc(self): + articles = Article.objects.all().order_by('-is_pinned', "-date_unique") + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', '-date_unique')) + page_1 = paginator.page(value=None) + self.assertListEqual(list(page_1), list(articles[:10])) + page_2 = paginator.page( + value=(page_1[-1].is_pinned, page_1[-1].date_unique)) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_3 = paginator.page( + value=(page_2[-1].is_pinned, page_2[-1].date_unique)) + self.assertListEqual(list(page_3), list(articles[20:])) + + def test_next_desc_pinned(self): + articles = Article.objects.all().order_by('-is_pinned', "-date_unique") + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_pinned = True + a.save() + for a in articles[20:]: + a.is_pinned = True + a.save() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', '-date_unique')) + page_1 = paginator.page(value=None) + self.assertListEqual(list(page_1), list(articles[:10])) + page_2 = paginator.page( + value=(page_1[-1].is_pinned, page_1[-1].date_unique)) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_3 = paginator.page( + value=(page_2[-1].is_pinned, page_2[-1].date_unique)) + self.assertListEqual(list(page_3), list(articles[20:])) + + def test_prev_desc(self): + articles = Article.objects.all().order_by('-is_pinned', "-date_unique") + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', '-date_unique')) + page_2 = paginator.page( + value=(list(articles)[20].is_pinned, list(articles)[20].date_unique), + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_1 = paginator.page( + value=(page_2[0].is_pinned, page_2[0].date_unique), + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_1), list(articles[:10])) + + def test_prev_desc_is_pinned(self): + articles = Article.objects.all().order_by('-is_pinned', "-date_unique") + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_pinned = True + a.save() + for a in articles[20:]: + a.is_pinned = True + a.save() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', '-date_unique')) + page_2 = paginator.page( + value=(list(articles)[20].is_pinned, list(articles)[20].date_unique), + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_1 = paginator.page( + value=(page_2[0].is_pinned, page_2[0].date_unique), + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_1), list(articles[:10])) + + def test_next_asc_desc(self): + articles = Article.objects.all().order_by('is_pinned', "-date_unique") + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_pinned = True + a.save() + for a in articles[20:]: + a.is_pinned = True + a.save() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('is_pinned', '-date_unique')) + page_1 = paginator.page(value=None) + self.assertListEqual(list(page_1), list(articles[:10])) + page_2 = paginator.page( + value=(page_1[-1].is_pinned, page_1[-1].date_unique)) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_3 = paginator.page( + value=(page_2[-1].is_pinned, page_2[-1].date_unique)) + self.assertListEqual(list(page_3), list(articles[20:])) + + def test_next_desc_asc(self): + articles = Article.objects.all().order_by('-is_pinned', "date_unique") + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_pinned = True + a.save() + for a in articles[20:]: + a.is_pinned = True + a.save() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', 'date_unique')) + page_1 = paginator.page(value=None) + self.assertListEqual(list(page_1), list(articles[:10])) + page_2 = paginator.page( + value=(page_1[-1].is_pinned, page_1[-1].date_unique)) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_3 = paginator.page( + value=(page_2[-1].is_pinned, page_2[-1].date_unique)) + self.assertListEqual(list(page_3), list(articles[20:])) + + def test_prev_asc_desc(self): + articles = Article.objects.all().order_by('is_pinned', "-date_unique") + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_pinned = True + a.save() + for a in articles[20:]: + a.is_pinned = True + a.save() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('is_pinned', '-date_unique')) + page_2 = paginator.page( + value=(list(articles)[20].is_pinned, list(articles)[20].date_unique), + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_1 = paginator.page( + value=(page_2[0].is_pinned, page_2[0].date_unique), + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_1), list(articles[:10])) + + def test_prev_desc_asc(self): + articles = Article.objects.all().order_by('-is_pinned', "date_unique") + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_pinned = True + a.save() + for a in articles[20:]: + a.is_pinned = True + a.save() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', 'date_unique')) + page_2 = paginator.page( + value=(list(articles)[20].is_pinned, list(articles)[20].date_unique), + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_1 = paginator.page( + value=(page_2[0].is_pinned, page_2[0].date_unique), + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_1), list(articles[:10])) + + def test_next_desc_non_unique(self): + articles = Article.objects.all().order_by('-is_pinned', "-date", "-pk") + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', '-date')) + page_1 = paginator.page(value=None, pk=None) + self.assertListEqual(list(page_1), list(articles[:10])) + page_2 = paginator.page( + value=(page_1[-1].is_pinned, page_1[-1].date), + pk=page_1[-1].pk) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_3 = paginator.page( + value=(page_2[-1].is_pinned, page_2[-1].date), + pk=page_2[-1].pk) + self.assertListEqual(list(page_3), list(articles[20:])) + + def test_next_desc_non_unique_pinned(self): + articles = Article.objects.all().order_by('-is_pinned', "-date", "-pk") + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_pinned = True + a.save() + for a in articles[20:]: + a.is_pinned = True + a.save() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', '-date')) + page_1 = paginator.page(value=None, pk=None) + self.assertListEqual(list(page_1), list(articles[:10])) + page_2 = paginator.page( + value=(page_1[-1].is_pinned, page_1[-1].date), + pk=page_1[-1].pk) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_3 = paginator.page( + value=(page_2[-1].is_pinned, page_2[-1].date), + pk=page_2[-1].pk) + self.assertListEqual(list(page_3), list(articles[20:])) + + def test_prev_desc_non_unique(self): + articles = Article.objects.all().order_by('-is_pinned', "-date", "-pk") + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', '-date')) + page_2 = paginator.page( + value=(list(articles)[20].is_pinned, list(articles)[20].date), + pk=list(articles)[20].pk, + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_1 = paginator.page( + value=(page_2[0].is_pinned, page_2[0].date), + pk=page_2[0].pk, + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_1), list(articles[:10])) + + def test_prev_desc_non_unique_is_pinned(self): + articles = Article.objects.all().order_by('-is_pinned', "-date", "-pk") + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_pinned = True + a.save() + for a in articles[20:]: + a.is_pinned = True + a.save() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', '-date')) + page_2 = paginator.page( + value=(list(articles)[20].is_pinned, list(articles)[20].date), + pk=list(articles)[20].pk, + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_1 = paginator.page( + value=(page_2[0].is_pinned, page_2[0].date), + pk=page_2[0].pk, + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_1), list(articles[:10])) + + def test_next_asc_desc_non_unique(self): + articles = Article.objects.all().order_by('is_pinned', "-date", "-pk") + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_pinned = True + a.save() + for a in articles[20:]: + a.is_pinned = True + a.save() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('is_pinned', '-date')) + page_1 = paginator.page(value=None, pk=None) + self.assertListEqual(list(page_1), list(articles[:10])) + page_2 = paginator.page( + value=(page_1[-1].is_pinned, page_1[-1].date), + pk=page_1[-1].pk) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_3 = paginator.page( + value=(page_2[-1].is_pinned, page_2[-1].date), + pk=page_2[-1].pk) + self.assertListEqual(list(page_3), list(articles[20:])) + + def test_next_desc_asc_non_unique(self): + articles = Article.objects.all().order_by('-is_pinned', "date", "pk") + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_pinned = True + a.save() + for a in articles[20:]: + a.is_pinned = True + a.save() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', 'date')) + page_1 = paginator.page(value=None, pk=None) + self.assertListEqual(list(page_1), list(articles[:10])) + page_2 = paginator.page( + value=(page_1[-1].is_pinned, page_1[-1].date), + pk=page_1[-1].pk) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_3 = paginator.page( + value=(page_2[-1].is_pinned, page_2[-1].date), + pk=page_2[-1].pk) + self.assertListEqual(list(page_3), list(articles[20:])) + + def test_prev_asc_desc_non_unique(self): + articles = Article.objects.all().order_by('is_pinned', "-date", "-pk") + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_pinned = True + a.save() + for a in articles[20:]: + a.is_pinned = True + a.save() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('is_pinned', '-date')) + page_2 = paginator.page( + value=(list(articles)[20].is_pinned, list(articles)[20].date), + pk=list(articles)[20].pk, + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_1 = paginator.page( + value=(page_2[0].is_pinned, page_2[0].date), + pk=page_2[0].pk, + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_1), list(articles[:10])) + + def test_prev_desc_asc_non_unique(self): + articles = Article.objects.all().order_by('-is_pinned', "date", "pk") + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_pinned = True + a.save() + for a in articles[20:]: + a.is_pinned = True + a.save() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', 'date')) + page_2 = paginator.page( + value=(list(articles)[20].is_pinned, list(articles)[20].date), + pk=list(articles)[20].pk, + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_1 = paginator.page( + value=(page_2[0].is_pinned, page_2[0].date), + pk=page_2[0].pk, + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_1), list(articles[:10])) + + +class Paginator3FieldsTest(TestCase): + + def setUp(self): + date = timezone.now() + for i in range(25): + seconds = datetime.timedelta(seconds=i) + Article.objects.create( + title="%s" % i, date=date, date_unique=date + seconds) + + def test_next_desc(self): + articles = Article.objects.all().order_by( + '-is_pinned', '-is_sticky', "-date_unique") + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', '-is_sticky', '-date_unique')) + page_1 = paginator.page(value=None) + self.assertListEqual(list(page_1), list(articles[:10])) + page_2 = paginator.page( + value=(page_1[-1].is_pinned, page_1[-1].is_sticky, page_1[-1].date_unique)) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_3 = paginator.page( + value=(page_2[-1].is_pinned, page_2[-1].is_sticky, page_2[-1].date_unique)) + self.assertListEqual(list(page_3), list(articles[20:])) + + def test_next_desc_sticky(self): + articles = Article.objects.all().order_by( + '-is_pinned', '-is_sticky', "-date_unique") + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_sticky = True + a.save() + for a in articles[20:]: + a.is_pinned = True + a.save() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', '-is_sticky', '-date_unique')) + page_1 = paginator.page(value=None) + self.assertListEqual(list(page_1), list(articles[:10])) + page_2 = paginator.page( + value=(page_1[-1].is_pinned, page_1[-1].is_sticky, page_1[-1].date_unique)) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_3 = paginator.page( + value=(page_2[-1].is_pinned, page_2[-1].is_sticky, page_2[-1].date_unique)) + self.assertListEqual(list(page_3), list(articles[20:])) + + def test_next_desc_sticky_pinned(self): + articles = Article.objects.all().order_by( + '-is_pinned', '-is_sticky', "-date_unique") + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_pinned = True + a.is_sticky = True + a.save() + for a in articles[20:]: + a.is_pinned = True + a.is_sticky = True + a.save() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', '-is_sticky', '-date_unique')) + page_1 = paginator.page(value=None) + self.assertListEqual(list(page_1), list(articles[:10])) + page_2 = paginator.page( + value=(page_1[-1].is_pinned, page_1[-1].is_sticky, page_1[-1].date_unique)) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_3 = paginator.page( + value=(page_2[-1].is_pinned, page_2[-1].is_sticky, page_2[-1].date_unique)) + self.assertListEqual(list(page_3), list(articles[20:])) + + def test_next_desc_asc_sticky_pinned(self): + articles = Article.objects.all().order_by( + '-is_pinned', 'is_sticky', "-date_unique") + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_pinned = True + a.is_sticky = True + a.save() + for a in articles[20:]: + a.is_pinned = True + a.is_sticky = True + a.save() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', 'is_sticky', '-date_unique')) + page_1 = paginator.page(value=None) + self.assertListEqual(list(page_1), list(articles[:10])) + page_2 = paginator.page( + value=(page_1[-1].is_pinned, page_1[-1].is_sticky, page_1[-1].date_unique)) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_3 = paginator.page( + value=(page_2[-1].is_pinned, page_2[-1].is_sticky, page_2[-1].date_unique)) + self.assertListEqual(list(page_3), list(articles[20:])) + + def test_prev_desc(self): + articles = Article.objects.all().order_by( + '-is_pinned', '-is_sticky', "-date_unique") + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', '-is_sticky', '-date_unique')) + page_2 = paginator.page( + value=( + list(articles)[20].is_pinned, + list(articles)[20].is_sticky, + list(articles)[20].date_unique), + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_1 = paginator.page( + value=( + page_2[0].is_pinned, + page_2[0].is_sticky, + page_2[0].date_unique), + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_1), list(articles[:10])) + + def test_prev_desc_sticky(self): + articles = Article.objects.all().order_by( + '-is_pinned', '-is_sticky', "-date_unique") + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_sticky = True + a.save() + for a in articles[20:]: + a.is_pinned = True + a.save() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', '-is_sticky', '-date_unique')) + page_2 = paginator.page( + value=( + list(articles)[20].is_pinned, + list(articles)[20].is_sticky, + list(articles)[20].date_unique), + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_1 = paginator.page( + value=( + page_2[0].is_pinned, + page_2[0].is_sticky, + page_2[0].date_unique), + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_1), list(articles[:10])) + + def test_prev_desc_sticky_pinned(self): + articles = Article.objects.all().order_by( + '-is_pinned', '-is_sticky', "-date_unique") + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_pinned = True + a.is_sticky = True + a.save() + for a in articles[20:]: + a.is_pinned = True + a.is_sticky = True + a.save() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', '-is_sticky', '-date_unique')) + page_2 = paginator.page( + value=( + list(articles)[20].is_pinned, + list(articles)[20].is_sticky, + list(articles)[20].date_unique), + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_1 = paginator.page( + value=( + page_2[0].is_pinned, + page_2[0].is_sticky, + page_2[0].date_unique), + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_1), list(articles[:10])) + + def test_prev_desc_asc_sticky_pinned(self): + articles = Article.objects.all().order_by( + '-is_pinned', 'is_sticky', "-date_unique") + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_pinned = True + a.is_sticky = True + a.save() + for a in articles[20:]: + a.is_pinned = True + a.is_sticky = True + a.save() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', 'is_sticky', '-date_unique')) + page_2 = paginator.page( + value=( + list(articles)[20].is_pinned, + list(articles)[20].is_sticky, + list(articles)[20].date_unique), + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_1 = paginator.page( + value=( + page_2[0].is_pinned, + page_2[0].is_sticky, + page_2[0].date_unique), + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_1), list(articles[:10])) + + def test_next_desc_non_unique(self): + articles = Article.objects.all().order_by( + '-is_pinned', '-is_sticky', "-date", "-pk") + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', '-is_sticky', '-date')) + page_1 = paginator.page(value=None, pk=None) + self.assertListEqual(list(page_1), list(articles[:10])) + page_2 = paginator.page( + value=(page_1[-1].is_pinned, page_1[-1].is_sticky, page_1[-1].date), + pk=page_1[-1].pk) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_3 = paginator.page( + value=(page_2[-1].is_pinned, page_2[-1].is_sticky, page_2[-1].date), + pk=page_2[-1].pk) + self.assertListEqual(list(page_3), list(articles[20:])) + + def test_next_desc_sticky_non_unique(self): + articles = Article.objects.all().order_by( + '-is_pinned', '-is_sticky', "-date", '-pk') + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_sticky = True + a.save() + for a in articles[20:]: + a.is_pinned = True + a.save() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', '-is_sticky', '-date')) + page_1 = paginator.page(value=None, pk=None) + self.assertListEqual(list(page_1), list(articles[:10])) + page_2 = paginator.page( + value=(page_1[-1].is_pinned, page_1[-1].is_sticky, page_1[-1].date), + pk=page_1[-1].pk) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_3 = paginator.page( + value=(page_2[-1].is_pinned, page_2[-1].is_sticky, page_2[-1].date), + pk=page_2[-1].pk) + self.assertListEqual(list(page_3), list(articles[20:])) + + def test_next_desc_sticky_pinned_non_unique(self): + articles = Article.objects.all().order_by( + '-is_pinned', '-is_sticky', "-date", '-pk') + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_pinned = True + a.is_sticky = True + a.save() + for a in articles[20:]: + a.is_pinned = True + a.is_sticky = True + a.save() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', '-is_sticky', '-date')) + page_1 = paginator.page(value=None, pk=None) + self.assertListEqual(list(page_1), list(articles[:10])) + page_2 = paginator.page( + value=(page_1[-1].is_pinned, page_1[-1].is_sticky, page_1[-1].date), + pk=page_1[-1].pk) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_3 = paginator.page( + value=(page_2[-1].is_pinned, page_2[-1].is_sticky, page_2[-1].date), + pk=page_2[-1].pk) + self.assertListEqual(list(page_3), list(articles[20:])) + + def test_next_desc_asc_sticky_pinned_non_unique(self): + articles = Article.objects.all().order_by( + '-is_pinned', 'is_sticky', "-date", '-pk') + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_pinned = True + a.is_sticky = True + a.save() + for a in articles[20:]: + a.is_pinned = True + a.is_sticky = True + a.save() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', 'is_sticky', '-date')) + page_1 = paginator.page(value=None, pk=None) + self.assertListEqual(list(page_1), list(articles[:10])) + page_2 = paginator.page( + value=(page_1[-1].is_pinned, page_1[-1].is_sticky, page_1[-1].date), + pk=page_1[-1].pk) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_3 = paginator.page( + value=(page_2[-1].is_pinned, page_2[-1].is_sticky, page_2[-1].date), + pk=page_2[-1].pk) + self.assertListEqual(list(page_3), list(articles[20:])) + + def test_prev_desc_non_unique(self): + articles = Article.objects.all().order_by( + '-is_pinned', '-is_sticky', "-date", '-pk') + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', '-is_sticky', '-date')) + page_2 = paginator.page( + value=( + list(articles)[20].is_pinned, + list(articles)[20].is_sticky, + list(articles)[20].date), + pk=list(articles)[20].pk, + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_1 = paginator.page( + value=( + page_2[0].is_pinned, + page_2[0].is_sticky, + page_2[0].date), + pk=page_2[0].pk, + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_1), list(articles[:10])) + + def test_prev_desc_sticky_non_unique(self): + articles = Article.objects.all().order_by( + '-is_pinned', '-is_sticky', "-date", '-pk') + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_sticky = True + a.save() + for a in articles[20:]: + a.is_pinned = True + a.save() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', '-is_sticky', '-date')) + page_2 = paginator.page( + value=( + list(articles)[20].is_pinned, + list(articles)[20].is_sticky, + list(articles)[20].date), + pk=list(articles)[20].pk, + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_1 = paginator.page( + value=( + page_2[0].is_pinned, + page_2[0].is_sticky, + page_2[0].date), + pk=page_2[0].pk, + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_1), list(articles[:10])) + + def test_prev_desc_sticky_pinned_non_unique(self): + articles = Article.objects.all().order_by( + '-is_pinned', '-is_sticky', "-date", '-pk') + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_pinned = True + a.is_sticky = True + a.save() + for a in articles[20:]: + a.is_pinned = True + a.is_sticky = True + a.save() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', '-is_sticky', '-date')) + page_2 = paginator.page( + value=( + list(articles)[20].is_pinned, + list(articles)[20].is_sticky, + list(articles)[20].date), + pk=list(articles)[20].pk, + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_1 = paginator.page( + value=( + page_2[0].is_pinned, + page_2[0].is_sticky, + page_2[0].date), + pk=page_2[0].pk, + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_1), list(articles[:10])) + + def test_prev_desc_asc_sticky_pinned_non_unique(self): + articles = Article.objects.all().order_by( + '-is_pinned', 'is_sticky', "-date", '-pk') + for a in articles[:5]: + a.is_pinned = True + a.save() + for a in articles[10:15]: + a.is_pinned = True + a.is_sticky = True + a.save() + for a in articles[20:]: + a.is_pinned = True + a.is_sticky = True + a.save() + paginator = SeekPaginator( + Article.objects.all(), + per_page=10, + lookup_field=('-is_pinned', 'is_sticky', '-date')) + page_2 = paginator.page( + value=( + list(articles)[20].is_pinned, + list(articles)[20].is_sticky, + list(articles)[20].date), + pk=list(articles)[20].pk, + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_2), list(articles[10:20])) + page_1 = paginator.page( + value=( + page_2[0].is_pinned, + page_2[0].is_sticky, + page_2[0].date), + pk=page_2[0].pk, + move_to=inf_paginator.PREV_PAGE) + self.assertListEqual(list(page_1), list(articles[:10]))