plain.models
Model your data and store it in a database.
# app/users/models.py
from plain import models
from plain.passwords.models import PasswordField
class User(models.Model):
email = models.EmailField()
password = PasswordField()
is_admin = models.BooleanField(default=False)
created_at = models.DateTimeField(auto_now_add=True)
def __str__(self):
return self.email
Create, update, and delete instances of your models:
from .models import User
# Create a new user
user = User.objects.create(
email="[email protected]",
password="password",
)
# Update a user
user.email = "[email protected]"
user.save()
# Delete a user
user.delete()
# Query for users
admin_users = User.objects.filter(is_admin=True)
Installation
# app/settings.py
INSTALLED_PACKAGES = [
...
"plain.models",
]
To connect to a database, you can provide a DATABASE_URL environment variable.
DATABASE_URL=postgresql://user:password@localhost:5432/dbname
Or you can manually define the DATABASES setting.
# app/settings.py
DATABASES = {
"default": {
"ENGINE": "plain.models.backends.postgresql",
"NAME": "dbname",
"USER": "user",
"PASSWORD": "password",
"HOST": "localhost",
"PORT": "5432",
}
}
Multiple backends are supported, including Postgres, MySQL, and SQLite.
Querying
Migrations
Fields
Validation
Indexes and constraints
Managers
Forms
1"""
2The main QuerySet implementation. This provides the public API for the ORM.
3"""
4
5import copy
6import operator
7import warnings
8from itertools import chain, islice
9
10import plain.runtime
11from plain import exceptions
12from plain.exceptions import ValidationError
13from plain.models import (
14 sql,
15 transaction,
16)
17from plain.models.constants import LOOKUP_SEP, OnConflict
18from plain.models.db import (
19 PLAIN_VERSION_PICKLE_KEY,
20 IntegrityError,
21 NotSupportedError,
22 connections,
23 router,
24)
25from plain.models.expressions import Case, F, Value, When
26from plain.models.fields import (
27 AutoField,
28 DateField,
29 DateTimeField,
30 Field,
31)
32from plain.models.functions import Cast, Trunc
33from plain.models.query_utils import FilteredRelation, Q
34from plain.models.sql.constants import CURSOR, GET_ITERATOR_CHUNK_SIZE
35from plain.models.utils import (
36 create_namedtuple_class,
37 resolve_callables,
38)
39from plain.utils import timezone
40from plain.utils.functional import cached_property, partition
41
42# The maximum number of results to fetch in a get() query.
43MAX_GET_RESULTS = 21
44
45# The maximum number of items to display in a QuerySet.__repr__
46REPR_OUTPUT_SIZE = 20
47
48
49class BaseIterable:
50 def __init__(
51 self, queryset, chunked_fetch=False, chunk_size=GET_ITERATOR_CHUNK_SIZE
52 ):
53 self.queryset = queryset
54 self.chunked_fetch = chunked_fetch
55 self.chunk_size = chunk_size
56
57
58class ModelIterable(BaseIterable):
59 """Iterable that yields a model instance for each row."""
60
61 def __iter__(self):
62 queryset = self.queryset
63 db = queryset.db
64 compiler = queryset.query.get_compiler(using=db)
65 # Execute the query. This will also fill compiler.select, klass_info,
66 # and annotations.
67 results = compiler.execute_sql(
68 chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size
69 )
70 select, klass_info, annotation_col_map = (
71 compiler.select,
72 compiler.klass_info,
73 compiler.annotation_col_map,
74 )
75 model_cls = klass_info["model"]
76 select_fields = klass_info["select_fields"]
77 model_fields_start, model_fields_end = select_fields[0], select_fields[-1] + 1
78 init_list = [
79 f[0].target.attname for f in select[model_fields_start:model_fields_end]
80 ]
81 related_populators = get_related_populators(klass_info, select, db)
82 known_related_objects = [
83 (
84 field,
85 related_objs,
86 operator.attrgetter(
87 *[
88 field.attname
89 if from_field == "self"
90 else queryset.model._meta.get_field(from_field).attname
91 for from_field in field.from_fields
92 ]
93 ),
94 )
95 for field, related_objs in queryset._known_related_objects.items()
96 ]
97 for row in compiler.results_iter(results):
98 obj = model_cls.from_db(
99 db, init_list, row[model_fields_start:model_fields_end]
100 )
101 for rel_populator in related_populators:
102 rel_populator.populate(row, obj)
103 if annotation_col_map:
104 for attr_name, col_pos in annotation_col_map.items():
105 setattr(obj, attr_name, row[col_pos])
106
107 # Add the known related objects to the model.
108 for field, rel_objs, rel_getter in known_related_objects:
109 # Avoid overwriting objects loaded by, e.g., select_related().
110 if field.is_cached(obj):
111 continue
112 rel_obj_id = rel_getter(obj)
113 try:
114 rel_obj = rel_objs[rel_obj_id]
115 except KeyError:
116 pass # May happen in qs1 | qs2 scenarios.
117 else:
118 setattr(obj, field.name, rel_obj)
119
120 yield obj
121
122
123class RawModelIterable(BaseIterable):
124 """
125 Iterable that yields a model instance for each row from a raw queryset.
126 """
127
128 def __iter__(self):
129 # Cache some things for performance reasons outside the loop.
130 db = self.queryset.db
131 query = self.queryset.query
132 connection = connections[db]
133 compiler = connection.ops.compiler("SQLCompiler")(query, connection, db)
134 query_iterator = iter(query)
135
136 try:
137 (
138 model_init_names,
139 model_init_pos,
140 annotation_fields,
141 ) = self.queryset.resolve_model_init_order()
142 model_cls = self.queryset.model
143 if model_cls._meta.pk.attname not in model_init_names:
144 raise exceptions.FieldDoesNotExist(
145 "Raw query must include the primary key"
146 )
147 fields = [self.queryset.model_fields.get(c) for c in self.queryset.columns]
148 converters = compiler.get_converters(
149 [f.get_col(f.model._meta.db_table) if f else None for f in fields]
150 )
151 if converters:
152 query_iterator = compiler.apply_converters(query_iterator, converters)
153 for values in query_iterator:
154 # Associate fields to values
155 model_init_values = [values[pos] for pos in model_init_pos]
156 instance = model_cls.from_db(db, model_init_names, model_init_values)
157 if annotation_fields:
158 for column, pos in annotation_fields:
159 setattr(instance, column, values[pos])
160 yield instance
161 finally:
162 # Done iterating the Query. If it has its own cursor, close it.
163 if hasattr(query, "cursor") and query.cursor:
164 query.cursor.close()
165
166
167class ValuesIterable(BaseIterable):
168 """
169 Iterable returned by QuerySet.values() that yields a dict for each row.
170 """
171
172 def __iter__(self):
173 queryset = self.queryset
174 query = queryset.query
175 compiler = query.get_compiler(queryset.db)
176
177 # extra(select=...) cols are always at the start of the row.
178 names = [
179 *query.extra_select,
180 *query.values_select,
181 *query.annotation_select,
182 ]
183 indexes = range(len(names))
184 for row in compiler.results_iter(
185 chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size
186 ):
187 yield {names[i]: row[i] for i in indexes}
188
189
190class ValuesListIterable(BaseIterable):
191 """
192 Iterable returned by QuerySet.values_list(flat=False) that yields a tuple
193 for each row.
194 """
195
196 def __iter__(self):
197 queryset = self.queryset
198 query = queryset.query
199 compiler = query.get_compiler(queryset.db)
200
201 if queryset._fields:
202 # extra(select=...) cols are always at the start of the row.
203 names = [
204 *query.extra_select,
205 *query.values_select,
206 *query.annotation_select,
207 ]
208 fields = [
209 *queryset._fields,
210 *(f for f in query.annotation_select if f not in queryset._fields),
211 ]
212 if fields != names:
213 # Reorder according to fields.
214 index_map = {name: idx for idx, name in enumerate(names)}
215 rowfactory = operator.itemgetter(*[index_map[f] for f in fields])
216 return map(
217 rowfactory,
218 compiler.results_iter(
219 chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size
220 ),
221 )
222 return compiler.results_iter(
223 tuple_expected=True,
224 chunked_fetch=self.chunked_fetch,
225 chunk_size=self.chunk_size,
226 )
227
228
229class NamedValuesListIterable(ValuesListIterable):
230 """
231 Iterable returned by QuerySet.values_list(named=True) that yields a
232 namedtuple for each row.
233 """
234
235 def __iter__(self):
236 queryset = self.queryset
237 if queryset._fields:
238 names = queryset._fields
239 else:
240 query = queryset.query
241 names = [
242 *query.extra_select,
243 *query.values_select,
244 *query.annotation_select,
245 ]
246 tuple_class = create_namedtuple_class(*names)
247 new = tuple.__new__
248 for row in super().__iter__():
249 yield new(tuple_class, row)
250
251
252class FlatValuesListIterable(BaseIterable):
253 """
254 Iterable returned by QuerySet.values_list(flat=True) that yields single
255 values.
256 """
257
258 def __iter__(self):
259 queryset = self.queryset
260 compiler = queryset.query.get_compiler(queryset.db)
261 for row in compiler.results_iter(
262 chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size
263 ):
264 yield row[0]
265
266
267class QuerySet:
268 """Represent a lazy database lookup for a set of objects."""
269
270 def __init__(self, model=None, query=None, using=None, hints=None):
271 self.model = model
272 self._db = using
273 self._hints = hints or {}
274 self._query = query or sql.Query(self.model)
275 self._result_cache = None
276 self._sticky_filter = False
277 self._for_write = False
278 self._prefetch_related_lookups = ()
279 self._prefetch_done = False
280 self._known_related_objects = {} # {rel_field: {pk: rel_obj}}
281 self._iterable_class = ModelIterable
282 self._fields = None
283 self._defer_next_filter = False
284 self._deferred_filter = None
285
286 @property
287 def query(self):
288 if self._deferred_filter:
289 negate, args, kwargs = self._deferred_filter
290 self._filter_or_exclude_inplace(negate, args, kwargs)
291 self._deferred_filter = None
292 return self._query
293
294 @query.setter
295 def query(self, value):
296 if value.values_select:
297 self._iterable_class = ValuesIterable
298 self._query = value
299
300 def as_manager(cls):
301 # Address the circular dependency between `Queryset` and `Manager`.
302 from plain.models.manager import Manager
303
304 manager = Manager.from_queryset(cls)()
305 manager._built_with_as_manager = True
306 return manager
307
308 as_manager.queryset_only = True
309 as_manager = classmethod(as_manager)
310
311 ########################
312 # PYTHON MAGIC METHODS #
313 ########################
314
315 def __deepcopy__(self, memo):
316 """Don't populate the QuerySet's cache."""
317 obj = self.__class__()
318 for k, v in self.__dict__.items():
319 if k == "_result_cache":
320 obj.__dict__[k] = None
321 else:
322 obj.__dict__[k] = copy.deepcopy(v, memo)
323 return obj
324
325 def __getstate__(self):
326 # Force the cache to be fully populated.
327 self._fetch_all()
328 return {**self.__dict__, PLAIN_VERSION_PICKLE_KEY: plain.runtime.__version__}
329
330 def __setstate__(self, state):
331 pickled_version = state.get(PLAIN_VERSION_PICKLE_KEY)
332 if pickled_version:
333 if pickled_version != plain.runtime.__version__:
334 warnings.warn(
335 f"Pickled queryset instance's Plain version {pickled_version} does not "
336 f"match the current version {plain.runtime.__version__}.",
337 RuntimeWarning,
338 stacklevel=2,
339 )
340 else:
341 warnings.warn(
342 "Pickled queryset instance's Plain version is not specified.",
343 RuntimeWarning,
344 stacklevel=2,
345 )
346 self.__dict__.update(state)
347
348 def __repr__(self):
349 data = list(self[: REPR_OUTPUT_SIZE + 1])
350 if len(data) > REPR_OUTPUT_SIZE:
351 data[-1] = "...(remaining elements truncated)..."
352 return f"<{self.__class__.__name__} {data!r}>"
353
354 def __len__(self):
355 self._fetch_all()
356 return len(self._result_cache)
357
358 def __iter__(self):
359 """
360 The queryset iterator protocol uses three nested iterators in the
361 default case:
362 1. sql.compiler.execute_sql()
363 - Returns 100 rows at time (constants.GET_ITERATOR_CHUNK_SIZE)
364 using cursor.fetchmany(). This part is responsible for
365 doing some column masking, and returning the rows in chunks.
366 2. sql.compiler.results_iter()
367 - Returns one row at time. At this point the rows are still just
368 tuples. In some cases the return values are converted to
369 Python values at this location.
370 3. self.iterator()
371 - Responsible for turning the rows into model objects.
372 """
373 self._fetch_all()
374 return iter(self._result_cache)
375
376 def __bool__(self):
377 self._fetch_all()
378 return bool(self._result_cache)
379
380 def __getitem__(self, k):
381 """Retrieve an item or slice from the set of results."""
382 if not isinstance(k, int | slice):
383 raise TypeError(
384 f"QuerySet indices must be integers or slices, not {type(k).__name__}."
385 )
386 if (isinstance(k, int) and k < 0) or (
387 isinstance(k, slice)
388 and (
389 (k.start is not None and k.start < 0)
390 or (k.stop is not None and k.stop < 0)
391 )
392 ):
393 raise ValueError("Negative indexing is not supported.")
394
395 if self._result_cache is not None:
396 return self._result_cache[k]
397
398 if isinstance(k, slice):
399 qs = self._chain()
400 if k.start is not None:
401 start = int(k.start)
402 else:
403 start = None
404 if k.stop is not None:
405 stop = int(k.stop)
406 else:
407 stop = None
408 qs.query.set_limits(start, stop)
409 return list(qs)[:: k.step] if k.step else qs
410
411 qs = self._chain()
412 qs.query.set_limits(k, k + 1)
413 qs._fetch_all()
414 return qs._result_cache[0]
415
416 def __class_getitem__(cls, *args, **kwargs):
417 return cls
418
419 def __and__(self, other):
420 self._check_operator_queryset(other, "&")
421 self._merge_sanity_check(other)
422 if isinstance(other, EmptyQuerySet):
423 return other
424 if isinstance(self, EmptyQuerySet):
425 return self
426 combined = self._chain()
427 combined._merge_known_related_objects(other)
428 combined.query.combine(other.query, sql.AND)
429 return combined
430
431 def __or__(self, other):
432 self._check_operator_queryset(other, "|")
433 self._merge_sanity_check(other)
434 if isinstance(self, EmptyQuerySet):
435 return other
436 if isinstance(other, EmptyQuerySet):
437 return self
438 query = (
439 self
440 if self.query.can_filter()
441 else self.model._base_manager.filter(pk__in=self.values("pk"))
442 )
443 combined = query._chain()
444 combined._merge_known_related_objects(other)
445 if not other.query.can_filter():
446 other = other.model._base_manager.filter(pk__in=other.values("pk"))
447 combined.query.combine(other.query, sql.OR)
448 return combined
449
450 def __xor__(self, other):
451 self._check_operator_queryset(other, "^")
452 self._merge_sanity_check(other)
453 if isinstance(self, EmptyQuerySet):
454 return other
455 if isinstance(other, EmptyQuerySet):
456 return self
457 query = (
458 self
459 if self.query.can_filter()
460 else self.model._base_manager.filter(pk__in=self.values("pk"))
461 )
462 combined = query._chain()
463 combined._merge_known_related_objects(other)
464 if not other.query.can_filter():
465 other = other.model._base_manager.filter(pk__in=other.values("pk"))
466 combined.query.combine(other.query, sql.XOR)
467 return combined
468
469 ####################################
470 # METHODS THAT DO DATABASE QUERIES #
471 ####################################
472
473 def _iterator(self, use_chunked_fetch, chunk_size):
474 iterable = self._iterable_class(
475 self,
476 chunked_fetch=use_chunked_fetch,
477 chunk_size=chunk_size or 2000,
478 )
479 if not self._prefetch_related_lookups or chunk_size is None:
480 yield from iterable
481 return
482
483 iterator = iter(iterable)
484 while results := list(islice(iterator, chunk_size)):
485 prefetch_related_objects(results, *self._prefetch_related_lookups)
486 yield from results
487
488 def iterator(self, chunk_size=None):
489 """
490 An iterator over the results from applying this QuerySet to the
491 database. chunk_size must be provided for QuerySets that prefetch
492 related objects. Otherwise, a default chunk_size of 2000 is supplied.
493 """
494 if chunk_size is None:
495 if self._prefetch_related_lookups:
496 raise ValueError(
497 "chunk_size must be provided when using QuerySet.iterator() after "
498 "prefetch_related()."
499 )
500 elif chunk_size <= 0:
501 raise ValueError("Chunk size must be strictly positive.")
502 use_chunked_fetch = not connections[self.db].settings_dict.get(
503 "DISABLE_SERVER_SIDE_CURSORS"
504 )
505 return self._iterator(use_chunked_fetch, chunk_size)
506
507 def aggregate(self, *args, **kwargs):
508 """
509 Return a dictionary containing the calculations (aggregation)
510 over the current queryset.
511
512 If args is present the expression is passed as a kwarg using
513 the Aggregate object's default alias.
514 """
515 if self.query.distinct_fields:
516 raise NotImplementedError("aggregate() + distinct(fields) not implemented.")
517 self._validate_values_are_expressions(
518 (*args, *kwargs.values()), method_name="aggregate"
519 )
520 for arg in args:
521 # The default_alias property raises TypeError if default_alias
522 # can't be set automatically or AttributeError if it isn't an
523 # attribute.
524 try:
525 arg.default_alias
526 except (AttributeError, TypeError):
527 raise TypeError("Complex aggregates require an alias")
528 kwargs[arg.default_alias] = arg
529
530 return self.query.chain().get_aggregation(self.db, kwargs)
531
532 def count(self):
533 """
534 Perform a SELECT COUNT() and return the number of records as an
535 integer.
536
537 If the QuerySet is already fully cached, return the length of the
538 cached results set to avoid multiple SELECT COUNT(*) calls.
539 """
540 if self._result_cache is not None:
541 return len(self._result_cache)
542
543 return self.query.get_count(using=self.db)
544
545 def get(self, *args, **kwargs):
546 """
547 Perform the query and return a single object matching the given
548 keyword arguments.
549 """
550 if self.query.combinator and (args or kwargs):
551 raise NotSupportedError(
552 f"Calling QuerySet.get(...) with filters after {self.query.combinator}() is not "
553 "supported."
554 )
555 clone = self._chain() if self.query.combinator else self.filter(*args, **kwargs)
556 if self.query.can_filter() and not self.query.distinct_fields:
557 clone = clone.order_by()
558 limit = None
559 if (
560 not clone.query.select_for_update
561 or connections[clone.db].features.supports_select_for_update_with_limit
562 ):
563 limit = MAX_GET_RESULTS
564 clone.query.set_limits(high=limit)
565 num = len(clone)
566 if num == 1:
567 return clone._result_cache[0]
568 if not num:
569 raise self.model.DoesNotExist(
570 f"{self.model._meta.object_name} matching query does not exist."
571 )
572 raise self.model.MultipleObjectsReturned(
573 "get() returned more than one {} -- it returned {}!".format(
574 self.model._meta.object_name,
575 num if not limit or num < limit else "more than %s" % (limit - 1),
576 )
577 )
578
579 def create(self, **kwargs):
580 """
581 Create a new object with the given kwargs, saving it to the database
582 and returning the created object.
583 """
584 obj = self.model(**kwargs)
585 self._for_write = True
586 obj.save(force_insert=True, using=self.db)
587 return obj
588
589 def _prepare_for_bulk_create(self, objs):
590 for obj in objs:
591 if obj.pk is None:
592 # Populate new PK values.
593 obj.pk = obj._meta.pk.get_pk_value_on_save(obj)
594 obj._prepare_related_fields_for_save(operation_name="bulk_create")
595
596 def _check_bulk_create_options(
597 self, update_conflicts, update_fields, unique_fields
598 ):
599 db_features = connections[self.db].features
600 if update_conflicts:
601 if not db_features.supports_update_conflicts:
602 raise NotSupportedError(
603 "This database backend does not support updating conflicts."
604 )
605 if not update_fields:
606 raise ValueError(
607 "Fields that will be updated when a row insertion fails "
608 "on conflicts must be provided."
609 )
610 if unique_fields and not db_features.supports_update_conflicts_with_target:
611 raise NotSupportedError(
612 "This database backend does not support updating "
613 "conflicts with specifying unique fields that can trigger "
614 "the upsert."
615 )
616 if not unique_fields and db_features.supports_update_conflicts_with_target:
617 raise ValueError(
618 "Unique fields that can trigger the upsert must be provided."
619 )
620 # Updating primary keys and non-concrete fields is forbidden.
621 if any(not f.concrete or f.many_to_many for f in update_fields):
622 raise ValueError(
623 "bulk_create() can only be used with concrete fields in "
624 "update_fields."
625 )
626 if any(f.primary_key for f in update_fields):
627 raise ValueError(
628 "bulk_create() cannot be used with primary keys in update_fields."
629 )
630 if unique_fields:
631 if any(not f.concrete or f.many_to_many for f in unique_fields):
632 raise ValueError(
633 "bulk_create() can only be used with concrete fields "
634 "in unique_fields."
635 )
636 return OnConflict.UPDATE
637 return None
638
639 def bulk_create(
640 self,
641 objs,
642 batch_size=None,
643 update_conflicts=False,
644 update_fields=None,
645 unique_fields=None,
646 ):
647 """
648 Insert each of the instances into the database. Do *not* call
649 save() on each of the instances, and do not set the primary key attribute if it is an
650 autoincrement field (except if features.can_return_rows_from_bulk_insert=True).
651 Multi-table models are not supported.
652 """
653 # When you bulk insert you don't get the primary keys back (if it's an
654 # autoincrement, except if can_return_rows_from_bulk_insert=True), so
655 # you can't insert into the child tables which references this. There
656 # are two workarounds:
657 # 1) This could be implemented if you didn't have an autoincrement pk
658 # 2) You could do it by doing O(n) normal inserts into the parent
659 # tables to get the primary keys back and then doing a single bulk
660 # insert into the childmost table.
661 # We currently set the primary keys on the objects when using
662 # PostgreSQL via the RETURNING ID clause. It should be possible for
663 # Oracle as well, but the semantics for extracting the primary keys is
664 # trickier so it's not done yet.
665 if batch_size is not None and batch_size <= 0:
666 raise ValueError("Batch size must be a positive integer.")
667
668 if not objs:
669 return objs
670 opts = self.model._meta
671 if unique_fields:
672 # Primary key is allowed in unique_fields.
673 unique_fields = [
674 self.model._meta.get_field(opts.pk.name if name == "pk" else name)
675 for name in unique_fields
676 ]
677 if update_fields:
678 update_fields = [self.model._meta.get_field(name) for name in update_fields]
679 on_conflict = self._check_bulk_create_options(
680 update_conflicts,
681 update_fields,
682 unique_fields,
683 )
684 self._for_write = True
685 fields = opts.concrete_fields
686 objs = list(objs)
687 self._prepare_for_bulk_create(objs)
688 with transaction.atomic(using=self.db, savepoint=False):
689 objs_with_pk, objs_without_pk = partition(lambda o: o.pk is None, objs)
690 if objs_with_pk:
691 returned_columns = self._batched_insert(
692 objs_with_pk,
693 fields,
694 batch_size,
695 on_conflict=on_conflict,
696 update_fields=update_fields,
697 unique_fields=unique_fields,
698 )
699 for obj_with_pk, results in zip(objs_with_pk, returned_columns):
700 for result, field in zip(results, opts.db_returning_fields):
701 if field != opts.pk:
702 setattr(obj_with_pk, field.attname, result)
703 for obj_with_pk in objs_with_pk:
704 obj_with_pk._state.adding = False
705 obj_with_pk._state.db = self.db
706 if objs_without_pk:
707 fields = [f for f in fields if not isinstance(f, AutoField)]
708 returned_columns = self._batched_insert(
709 objs_without_pk,
710 fields,
711 batch_size,
712 on_conflict=on_conflict,
713 update_fields=update_fields,
714 unique_fields=unique_fields,
715 )
716 connection = connections[self.db]
717 if (
718 connection.features.can_return_rows_from_bulk_insert
719 and on_conflict is None
720 ):
721 assert len(returned_columns) == len(objs_without_pk)
722 for obj_without_pk, results in zip(objs_without_pk, returned_columns):
723 for result, field in zip(results, opts.db_returning_fields):
724 setattr(obj_without_pk, field.attname, result)
725 obj_without_pk._state.adding = False
726 obj_without_pk._state.db = self.db
727
728 return objs
729
730 def bulk_update(self, objs, fields, batch_size=None):
731 """
732 Update the given fields in each of the given objects in the database.
733 """
734 if batch_size is not None and batch_size <= 0:
735 raise ValueError("Batch size must be a positive integer.")
736 if not fields:
737 raise ValueError("Field names must be given to bulk_update().")
738 objs = tuple(objs)
739 if any(obj.pk is None for obj in objs):
740 raise ValueError("All bulk_update() objects must have a primary key set.")
741 fields = [self.model._meta.get_field(name) for name in fields]
742 if any(not f.concrete or f.many_to_many for f in fields):
743 raise ValueError("bulk_update() can only be used with concrete fields.")
744 if any(f.primary_key for f in fields):
745 raise ValueError("bulk_update() cannot be used with primary key fields.")
746 if not objs:
747 return 0
748 for obj in objs:
749 obj._prepare_related_fields_for_save(
750 operation_name="bulk_update", fields=fields
751 )
752 # PK is used twice in the resulting update query, once in the filter
753 # and once in the WHEN. Each field will also have one CAST.
754 self._for_write = True
755 connection = connections[self.db]
756 max_batch_size = connection.ops.bulk_batch_size(["pk", "pk"] + fields, objs)
757 batch_size = min(batch_size, max_batch_size) if batch_size else max_batch_size
758 requires_casting = connection.features.requires_casted_case_in_updates
759 batches = (objs[i : i + batch_size] for i in range(0, len(objs), batch_size))
760 updates = []
761 for batch_objs in batches:
762 update_kwargs = {}
763 for field in fields:
764 when_statements = []
765 for obj in batch_objs:
766 attr = getattr(obj, field.attname)
767 if not hasattr(attr, "resolve_expression"):
768 attr = Value(attr, output_field=field)
769 when_statements.append(When(pk=obj.pk, then=attr))
770 case_statement = Case(*when_statements, output_field=field)
771 if requires_casting:
772 case_statement = Cast(case_statement, output_field=field)
773 update_kwargs[field.attname] = case_statement
774 updates.append(([obj.pk for obj in batch_objs], update_kwargs))
775 rows_updated = 0
776 queryset = self.using(self.db)
777 with transaction.atomic(using=self.db, savepoint=False):
778 for pks, update_kwargs in updates:
779 rows_updated += queryset.filter(pk__in=pks).update(**update_kwargs)
780 return rows_updated
781
782 def get_or_create(self, defaults=None, **kwargs):
783 """
784 Look up an object with the given kwargs, creating one if necessary.
785 Return a tuple of (object, created), where created is a boolean
786 specifying whether an object was created.
787 """
788 # The get() needs to be targeted at the write database in order
789 # to avoid potential transaction consistency problems.
790 self._for_write = True
791 try:
792 return self.get(**kwargs), False
793 except self.model.DoesNotExist:
794 params = self._extract_model_params(defaults, **kwargs)
795 # Try to create an object using passed params.
796 try:
797 with transaction.atomic(using=self.db):
798 params = dict(resolve_callables(params))
799 return self.create(**params), True
800 except (IntegrityError, ValidationError):
801 # Since create() also validates by default,
802 # we can get any kind of ValidationError here,
803 # or it can flow through and get an IntegrityError from the database.
804 # The main thing we're concerned about is uniqueness failures,
805 # but ValidationError could include other things too.
806 # In all cases though it should be fine to try the get() again
807 # and return an existing object.
808 try:
809 return self.get(**kwargs), False
810 except self.model.DoesNotExist:
811 pass
812 raise
813
814 def update_or_create(self, defaults=None, create_defaults=None, **kwargs):
815 """
816 Look up an object with the given kwargs, updating one with defaults
817 if it exists, otherwise create a new one. Optionally, an object can
818 be created with different values than defaults by using
819 create_defaults.
820 Return a tuple (object, created), where created is a boolean
821 specifying whether an object was created.
822 """
823 if create_defaults is None:
824 update_defaults = create_defaults = defaults or {}
825 else:
826 update_defaults = defaults or {}
827 self._for_write = True
828 with transaction.atomic(using=self.db):
829 # Lock the row so that a concurrent update is blocked until
830 # update_or_create() has performed its save.
831 obj, created = self.select_for_update().get_or_create(
832 create_defaults, **kwargs
833 )
834 if created:
835 return obj, created
836 for k, v in resolve_callables(update_defaults):
837 setattr(obj, k, v)
838
839 update_fields = set(update_defaults)
840 concrete_field_names = self.model._meta._non_pk_concrete_field_names
841 # update_fields does not support non-concrete fields.
842 if concrete_field_names.issuperset(update_fields):
843 # Add fields which are set on pre_save(), e.g. auto_now fields.
844 # This is to maintain backward compatibility as these fields
845 # are not updated unless explicitly specified in the
846 # update_fields list.
847 for field in self.model._meta.local_concrete_fields:
848 if not (
849 field.primary_key or field.__class__.pre_save is Field.pre_save
850 ):
851 update_fields.add(field.name)
852 if field.name != field.attname:
853 update_fields.add(field.attname)
854 obj.save(using=self.db, update_fields=update_fields)
855 else:
856 obj.save(using=self.db)
857 return obj, False
858
859 def _extract_model_params(self, defaults, **kwargs):
860 """
861 Prepare `params` for creating a model instance based on the given
862 kwargs; for use by get_or_create().
863 """
864 defaults = defaults or {}
865 params = {k: v for k, v in kwargs.items() if LOOKUP_SEP not in k}
866 params.update(defaults)
867 property_names = self.model._meta._property_names
868 invalid_params = []
869 for param in params:
870 try:
871 self.model._meta.get_field(param)
872 except exceptions.FieldDoesNotExist:
873 # It's okay to use a model's property if it has a setter.
874 if not (param in property_names and getattr(self.model, param).fset):
875 invalid_params.append(param)
876 if invalid_params:
877 raise exceptions.FieldError(
878 "Invalid field name(s) for model {}: '{}'.".format(
879 self.model._meta.object_name,
880 "', '".join(sorted(invalid_params)),
881 )
882 )
883 return params
884
885 def _earliest(self, *fields):
886 """
887 Return the earliest object according to fields (if given) or by the
888 model's Meta.get_latest_by.
889 """
890 if fields:
891 order_by = fields
892 else:
893 order_by = getattr(self.model._meta, "get_latest_by")
894 if order_by and not isinstance(order_by, tuple | list):
895 order_by = (order_by,)
896 if order_by is None:
897 raise ValueError(
898 "earliest() and latest() require either fields as positional "
899 "arguments or 'get_latest_by' in the model's Meta."
900 )
901 obj = self._chain()
902 obj.query.set_limits(high=1)
903 obj.query.clear_ordering(force=True)
904 obj.query.add_ordering(*order_by)
905 return obj.get()
906
907 def earliest(self, *fields):
908 if self.query.is_sliced:
909 raise TypeError("Cannot change a query once a slice has been taken.")
910 return self._earliest(*fields)
911
912 def latest(self, *fields):
913 """
914 Return the latest object according to fields (if given) or by the
915 model's Meta.get_latest_by.
916 """
917 if self.query.is_sliced:
918 raise TypeError("Cannot change a query once a slice has been taken.")
919 return self.reverse()._earliest(*fields)
920
921 def first(self):
922 """Return the first object of a query or None if no match is found."""
923 if self.ordered:
924 queryset = self
925 else:
926 self._check_ordering_first_last_queryset_aggregation(method="first")
927 queryset = self.order_by("pk")
928 for obj in queryset[:1]:
929 return obj
930
931 def last(self):
932 """Return the last object of a query or None if no match is found."""
933 if self.ordered:
934 queryset = self.reverse()
935 else:
936 self._check_ordering_first_last_queryset_aggregation(method="last")
937 queryset = self.order_by("-pk")
938 for obj in queryset[:1]:
939 return obj
940
941 def in_bulk(self, id_list=None, *, field_name="pk"):
942 """
943 Return a dictionary mapping each of the given IDs to the object with
944 that ID. If `id_list` isn't provided, evaluate the entire QuerySet.
945 """
946 if self.query.is_sliced:
947 raise TypeError("Cannot use 'limit' or 'offset' with in_bulk().")
948 opts = self.model._meta
949 unique_fields = [
950 constraint.fields[0]
951 for constraint in opts.total_unique_constraints
952 if len(constraint.fields) == 1
953 ]
954 if (
955 field_name != "pk"
956 and not opts.get_field(field_name).primary_key
957 and field_name not in unique_fields
958 and self.query.distinct_fields != (field_name,)
959 ):
960 raise ValueError(
961 f"in_bulk()'s field_name must be a unique field but {field_name!r} isn't."
962 )
963 if id_list is not None:
964 if not id_list:
965 return {}
966 filter_key = f"{field_name}__in"
967 batch_size = connections[self.db].features.max_query_params
968 id_list = tuple(id_list)
969 # If the database has a limit on the number of query parameters
970 # (e.g. SQLite), retrieve objects in batches if necessary.
971 if batch_size and batch_size < len(id_list):
972 qs = ()
973 for offset in range(0, len(id_list), batch_size):
974 batch = id_list[offset : offset + batch_size]
975 qs += tuple(self.filter(**{filter_key: batch}))
976 else:
977 qs = self.filter(**{filter_key: id_list})
978 else:
979 qs = self._chain()
980 return {getattr(obj, field_name): obj for obj in qs}
981
982 def delete(self):
983 """Delete the records in the current QuerySet."""
984 self._not_support_combined_queries("delete")
985 if self.query.is_sliced:
986 raise TypeError("Cannot use 'limit' or 'offset' with delete().")
987 if self.query.distinct or self.query.distinct_fields:
988 raise TypeError("Cannot call delete() after .distinct().")
989 if self._fields is not None:
990 raise TypeError("Cannot call delete() after .values() or .values_list()")
991
992 del_query = self._chain()
993
994 # The delete is actually 2 queries - one to find related objects,
995 # and one to delete. Make sure that the discovery of related
996 # objects is performed on the same database as the deletion.
997 del_query._for_write = True
998
999 # Disable non-supported fields.
1000 del_query.query.select_for_update = False
1001 del_query.query.select_related = False
1002 del_query.query.clear_ordering(force=True)
1003
1004 from plain.models.deletion import Collector
1005
1006 collector = Collector(using=del_query.db, origin=self)
1007 collector.collect(del_query)
1008 deleted, _rows_count = collector.delete()
1009
1010 # Clear the result cache, in case this QuerySet gets reused.
1011 self._result_cache = None
1012 return deleted, _rows_count
1013
1014 delete.queryset_only = True
1015
1016 def _raw_delete(self, using):
1017 """
1018 Delete objects found from the given queryset in single direct SQL
1019 query. No signals are sent and there is no protection for cascades.
1020 """
1021 query = self.query.clone()
1022 query.__class__ = sql.DeleteQuery
1023 cursor = query.get_compiler(using).execute_sql(CURSOR)
1024 if cursor:
1025 with cursor:
1026 return cursor.rowcount
1027 return 0
1028
1029 def update(self, **kwargs):
1030 """
1031 Update all elements in the current QuerySet, setting all the given
1032 fields to the appropriate values.
1033 """
1034 self._not_support_combined_queries("update")
1035 if self.query.is_sliced:
1036 raise TypeError("Cannot update a query once a slice has been taken.")
1037 self._for_write = True
1038 query = self.query.chain(sql.UpdateQuery)
1039 query.add_update_values(kwargs)
1040
1041 # Inline annotations in order_by(), if possible.
1042 new_order_by = []
1043 for col in query.order_by:
1044 alias = col
1045 descending = False
1046 if isinstance(alias, str) and alias.startswith("-"):
1047 alias = alias.removeprefix("-")
1048 descending = True
1049 if annotation := query.annotations.get(alias):
1050 if getattr(annotation, "contains_aggregate", False):
1051 raise exceptions.FieldError(
1052 f"Cannot update when ordering by an aggregate: {annotation}"
1053 )
1054 if descending:
1055 annotation = annotation.desc()
1056 new_order_by.append(annotation)
1057 else:
1058 new_order_by.append(col)
1059 query.order_by = tuple(new_order_by)
1060
1061 # Clear any annotations so that they won't be present in subqueries.
1062 query.annotations = {}
1063 with transaction.mark_for_rollback_on_error(using=self.db):
1064 rows = query.get_compiler(self.db).execute_sql(CURSOR)
1065 self._result_cache = None
1066 return rows
1067
1068 def _update(self, values):
1069 """
1070 A version of update() that accepts field objects instead of field names.
1071 Used primarily for model saving and not intended for use by general
1072 code (it requires too much poking around at model internals to be
1073 useful at that level).
1074 """
1075 if self.query.is_sliced:
1076 raise TypeError("Cannot update a query once a slice has been taken.")
1077 query = self.query.chain(sql.UpdateQuery)
1078 query.add_update_fields(values)
1079 # Clear any annotations so that they won't be present in subqueries.
1080 query.annotations = {}
1081 self._result_cache = None
1082 return query.get_compiler(self.db).execute_sql(CURSOR)
1083
1084 _update.queryset_only = False
1085
1086 def exists(self):
1087 """
1088 Return True if the QuerySet would have any results, False otherwise.
1089 """
1090 if self._result_cache is None:
1091 return self.query.has_results(using=self.db)
1092 return bool(self._result_cache)
1093
1094 def contains(self, obj):
1095 """
1096 Return True if the QuerySet contains the provided obj,
1097 False otherwise.
1098 """
1099 self._not_support_combined_queries("contains")
1100 if self._fields is not None:
1101 raise TypeError(
1102 "Cannot call QuerySet.contains() after .values() or .values_list()."
1103 )
1104 try:
1105 if obj._meta.concrete_model != self.model._meta.concrete_model:
1106 return False
1107 except AttributeError:
1108 raise TypeError("'obj' must be a model instance.")
1109 if obj.pk is None:
1110 raise ValueError("QuerySet.contains() cannot be used on unsaved objects.")
1111 if self._result_cache is not None:
1112 return obj in self._result_cache
1113 return self.filter(pk=obj.pk).exists()
1114
1115 def _prefetch_related_objects(self):
1116 # This method can only be called once the result cache has been filled.
1117 prefetch_related_objects(self._result_cache, *self._prefetch_related_lookups)
1118 self._prefetch_done = True
1119
1120 def explain(self, *, format=None, **options):
1121 """
1122 Runs an EXPLAIN on the SQL query this QuerySet would perform, and
1123 returns the results.
1124 """
1125 return self.query.explain(using=self.db, format=format, **options)
1126
1127 ##################################################
1128 # PUBLIC METHODS THAT RETURN A QUERYSET SUBCLASS #
1129 ##################################################
1130
1131 def raw(self, raw_query, params=(), translations=None, using=None):
1132 if using is None:
1133 using = self.db
1134 qs = RawQuerySet(
1135 raw_query,
1136 model=self.model,
1137 params=params,
1138 translations=translations,
1139 using=using,
1140 )
1141 qs._prefetch_related_lookups = self._prefetch_related_lookups[:]
1142 return qs
1143
1144 def _values(self, *fields, **expressions):
1145 clone = self._chain()
1146 if expressions:
1147 clone = clone.annotate(**expressions)
1148 clone._fields = fields
1149 clone.query.set_values(fields)
1150 return clone
1151
1152 def values(self, *fields, **expressions):
1153 fields += tuple(expressions)
1154 clone = self._values(*fields, **expressions)
1155 clone._iterable_class = ValuesIterable
1156 return clone
1157
1158 def values_list(self, *fields, flat=False, named=False):
1159 if flat and named:
1160 raise TypeError("'flat' and 'named' can't be used together.")
1161 if flat and len(fields) > 1:
1162 raise TypeError(
1163 "'flat' is not valid when values_list is called with more than one "
1164 "field."
1165 )
1166
1167 field_names = {f for f in fields if not hasattr(f, "resolve_expression")}
1168 _fields = []
1169 expressions = {}
1170 counter = 1
1171 for field in fields:
1172 if hasattr(field, "resolve_expression"):
1173 field_id_prefix = getattr(
1174 field, "default_alias", field.__class__.__name__.lower()
1175 )
1176 while True:
1177 field_id = field_id_prefix + str(counter)
1178 counter += 1
1179 if field_id not in field_names:
1180 break
1181 expressions[field_id] = field
1182 _fields.append(field_id)
1183 else:
1184 _fields.append(field)
1185
1186 clone = self._values(*_fields, **expressions)
1187 clone._iterable_class = (
1188 NamedValuesListIterable
1189 if named
1190 else FlatValuesListIterable
1191 if flat
1192 else ValuesListIterable
1193 )
1194 return clone
1195
1196 def dates(self, field_name, kind, order="ASC"):
1197 """
1198 Return a list of date objects representing all available dates for
1199 the given field_name, scoped to 'kind'.
1200 """
1201 if kind not in ("year", "month", "week", "day"):
1202 raise ValueError("'kind' must be one of 'year', 'month', 'week', or 'day'.")
1203 if order not in ("ASC", "DESC"):
1204 raise ValueError("'order' must be either 'ASC' or 'DESC'.")
1205 return (
1206 self.annotate(
1207 datefield=Trunc(field_name, kind, output_field=DateField()),
1208 plain_field=F(field_name),
1209 )
1210 .values_list("datefield", flat=True)
1211 .distinct()
1212 .filter(plain_field__isnull=False)
1213 .order_by(("-" if order == "DESC" else "") + "datefield")
1214 )
1215
1216 def datetimes(self, field_name, kind, order="ASC", tzinfo=None):
1217 """
1218 Return a list of datetime objects representing all available
1219 datetimes for the given field_name, scoped to 'kind'.
1220 """
1221 if kind not in ("year", "month", "week", "day", "hour", "minute", "second"):
1222 raise ValueError(
1223 "'kind' must be one of 'year', 'month', 'week', 'day', "
1224 "'hour', 'minute', or 'second'."
1225 )
1226 if order not in ("ASC", "DESC"):
1227 raise ValueError("'order' must be either 'ASC' or 'DESC'.")
1228
1229 if tzinfo is None:
1230 tzinfo = timezone.get_current_timezone()
1231
1232 return (
1233 self.annotate(
1234 datetimefield=Trunc(
1235 field_name,
1236 kind,
1237 output_field=DateTimeField(),
1238 tzinfo=tzinfo,
1239 ),
1240 plain_field=F(field_name),
1241 )
1242 .values_list("datetimefield", flat=True)
1243 .distinct()
1244 .filter(plain_field__isnull=False)
1245 .order_by(("-" if order == "DESC" else "") + "datetimefield")
1246 )
1247
1248 def none(self):
1249 """Return an empty QuerySet."""
1250 clone = self._chain()
1251 clone.query.set_empty()
1252 return clone
1253
1254 ##################################################################
1255 # PUBLIC METHODS THAT ALTER ATTRIBUTES AND RETURN A NEW QUERYSET #
1256 ##################################################################
1257
1258 def all(self):
1259 """
1260 Return a new QuerySet that is a copy of the current one. This allows a
1261 QuerySet to proxy for a model manager in some cases.
1262 """
1263 return self._chain()
1264
1265 def filter(self, *args, **kwargs):
1266 """
1267 Return a new QuerySet instance with the args ANDed to the existing
1268 set.
1269 """
1270 self._not_support_combined_queries("filter")
1271 return self._filter_or_exclude(False, args, kwargs)
1272
1273 def exclude(self, *args, **kwargs):
1274 """
1275 Return a new QuerySet instance with NOT (args) ANDed to the existing
1276 set.
1277 """
1278 self._not_support_combined_queries("exclude")
1279 return self._filter_or_exclude(True, args, kwargs)
1280
1281 def _filter_or_exclude(self, negate, args, kwargs):
1282 if (args or kwargs) and self.query.is_sliced:
1283 raise TypeError("Cannot filter a query once a slice has been taken.")
1284 clone = self._chain()
1285 if self._defer_next_filter:
1286 self._defer_next_filter = False
1287 clone._deferred_filter = negate, args, kwargs
1288 else:
1289 clone._filter_or_exclude_inplace(negate, args, kwargs)
1290 return clone
1291
1292 def _filter_or_exclude_inplace(self, negate, args, kwargs):
1293 if negate:
1294 self._query.add_q(~Q(*args, **kwargs))
1295 else:
1296 self._query.add_q(Q(*args, **kwargs))
1297
1298 def complex_filter(self, filter_obj):
1299 """
1300 Return a new QuerySet instance with filter_obj added to the filters.
1301
1302 filter_obj can be a Q object or a dictionary of keyword lookup
1303 arguments.
1304
1305 This exists to support framework features such as 'limit_choices_to',
1306 and usually it will be more natural to use other methods.
1307 """
1308 if isinstance(filter_obj, Q):
1309 clone = self._chain()
1310 clone.query.add_q(filter_obj)
1311 return clone
1312 else:
1313 return self._filter_or_exclude(False, args=(), kwargs=filter_obj)
1314
1315 def _combinator_query(self, combinator, *other_qs, all=False):
1316 # Clone the query to inherit the select list and everything
1317 clone = self._chain()
1318 # Clear limits and ordering so they can be reapplied
1319 clone.query.clear_ordering(force=True)
1320 clone.query.clear_limits()
1321 clone.query.combined_queries = (self.query,) + tuple(
1322 qs.query for qs in other_qs
1323 )
1324 clone.query.combinator = combinator
1325 clone.query.combinator_all = all
1326 return clone
1327
1328 def union(self, *other_qs, all=False):
1329 # If the query is an EmptyQuerySet, combine all nonempty querysets.
1330 if isinstance(self, EmptyQuerySet):
1331 qs = [q for q in other_qs if not isinstance(q, EmptyQuerySet)]
1332 if not qs:
1333 return self
1334 if len(qs) == 1:
1335 return qs[0]
1336 return qs[0]._combinator_query("union", *qs[1:], all=all)
1337 return self._combinator_query("union", *other_qs, all=all)
1338
1339 def intersection(self, *other_qs):
1340 # If any query is an EmptyQuerySet, return it.
1341 if isinstance(self, EmptyQuerySet):
1342 return self
1343 for other in other_qs:
1344 if isinstance(other, EmptyQuerySet):
1345 return other
1346 return self._combinator_query("intersection", *other_qs)
1347
1348 def difference(self, *other_qs):
1349 # If the query is an EmptyQuerySet, return it.
1350 if isinstance(self, EmptyQuerySet):
1351 return self
1352 return self._combinator_query("difference", *other_qs)
1353
1354 def select_for_update(self, nowait=False, skip_locked=False, of=(), no_key=False):
1355 """
1356 Return a new QuerySet instance that will select objects with a
1357 FOR UPDATE lock.
1358 """
1359 if nowait and skip_locked:
1360 raise ValueError("The nowait option cannot be used with skip_locked.")
1361 obj = self._chain()
1362 obj._for_write = True
1363 obj.query.select_for_update = True
1364 obj.query.select_for_update_nowait = nowait
1365 obj.query.select_for_update_skip_locked = skip_locked
1366 obj.query.select_for_update_of = of
1367 obj.query.select_for_no_key_update = no_key
1368 return obj
1369
1370 def select_related(self, *fields):
1371 """
1372 Return a new QuerySet instance that will select related objects.
1373
1374 If fields are specified, they must be ForeignKey fields and only those
1375 related objects are included in the selection.
1376
1377 If select_related(None) is called, clear the list.
1378 """
1379 self._not_support_combined_queries("select_related")
1380 if self._fields is not None:
1381 raise TypeError(
1382 "Cannot call select_related() after .values() or .values_list()"
1383 )
1384
1385 obj = self._chain()
1386 if fields == (None,):
1387 obj.query.select_related = False
1388 elif fields:
1389 obj.query.add_select_related(fields)
1390 else:
1391 obj.query.select_related = True
1392 return obj
1393
1394 def prefetch_related(self, *lookups):
1395 """
1396 Return a new QuerySet instance that will prefetch the specified
1397 Many-To-One and Many-To-Many related objects when the QuerySet is
1398 evaluated.
1399
1400 When prefetch_related() is called more than once, append to the list of
1401 prefetch lookups. If prefetch_related(None) is called, clear the list.
1402 """
1403 self._not_support_combined_queries("prefetch_related")
1404 clone = self._chain()
1405 if lookups == (None,):
1406 clone._prefetch_related_lookups = ()
1407 else:
1408 for lookup in lookups:
1409 if isinstance(lookup, Prefetch):
1410 lookup = lookup.prefetch_to
1411 lookup = lookup.split(LOOKUP_SEP, 1)[0]
1412 if lookup in self.query._filtered_relations:
1413 raise ValueError(
1414 "prefetch_related() is not supported with FilteredRelation."
1415 )
1416 clone._prefetch_related_lookups = clone._prefetch_related_lookups + lookups
1417 return clone
1418
1419 def annotate(self, *args, **kwargs):
1420 """
1421 Return a query set in which the returned objects have been annotated
1422 with extra data or aggregations.
1423 """
1424 self._not_support_combined_queries("annotate")
1425 return self._annotate(args, kwargs, select=True)
1426
1427 def alias(self, *args, **kwargs):
1428 """
1429 Return a query set with added aliases for extra data or aggregations.
1430 """
1431 self._not_support_combined_queries("alias")
1432 return self._annotate(args, kwargs, select=False)
1433
1434 def _annotate(self, args, kwargs, select=True):
1435 self._validate_values_are_expressions(
1436 args + tuple(kwargs.values()), method_name="annotate"
1437 )
1438 annotations = {}
1439 for arg in args:
1440 # The default_alias property may raise a TypeError.
1441 try:
1442 if arg.default_alias in kwargs:
1443 raise ValueError(
1444 f"The named annotation '{arg.default_alias}' conflicts with the "
1445 "default name for another annotation."
1446 )
1447 except TypeError:
1448 raise TypeError("Complex annotations require an alias")
1449 annotations[arg.default_alias] = arg
1450 annotations.update(kwargs)
1451
1452 clone = self._chain()
1453 names = self._fields
1454 if names is None:
1455 names = set(
1456 chain.from_iterable(
1457 (field.name, field.attname)
1458 if hasattr(field, "attname")
1459 else (field.name,)
1460 for field in self.model._meta.get_fields()
1461 )
1462 )
1463
1464 for alias, annotation in annotations.items():
1465 if alias in names:
1466 raise ValueError(
1467 f"The annotation '{alias}' conflicts with a field on the model."
1468 )
1469 if isinstance(annotation, FilteredRelation):
1470 clone.query.add_filtered_relation(annotation, alias)
1471 else:
1472 clone.query.add_annotation(
1473 annotation,
1474 alias,
1475 select=select,
1476 )
1477 for alias, annotation in clone.query.annotations.items():
1478 if alias in annotations and annotation.contains_aggregate:
1479 if clone._fields is None:
1480 clone.query.group_by = True
1481 else:
1482 clone.query.set_group_by()
1483 break
1484
1485 return clone
1486
1487 def order_by(self, *field_names):
1488 """Return a new QuerySet instance with the ordering changed."""
1489 if self.query.is_sliced:
1490 raise TypeError("Cannot reorder a query once a slice has been taken.")
1491 obj = self._chain()
1492 obj.query.clear_ordering(force=True, clear_default=False)
1493 obj.query.add_ordering(*field_names)
1494 return obj
1495
1496 def distinct(self, *field_names):
1497 """
1498 Return a new QuerySet instance that will select only distinct results.
1499 """
1500 self._not_support_combined_queries("distinct")
1501 if self.query.is_sliced:
1502 raise TypeError(
1503 "Cannot create distinct fields once a slice has been taken."
1504 )
1505 obj = self._chain()
1506 obj.query.add_distinct_fields(*field_names)
1507 return obj
1508
1509 def extra(
1510 self,
1511 select=None,
1512 where=None,
1513 params=None,
1514 tables=None,
1515 order_by=None,
1516 select_params=None,
1517 ):
1518 """Add extra SQL fragments to the query."""
1519 self._not_support_combined_queries("extra")
1520 if self.query.is_sliced:
1521 raise TypeError("Cannot change a query once a slice has been taken.")
1522 clone = self._chain()
1523 clone.query.add_extra(select, select_params, where, params, tables, order_by)
1524 return clone
1525
1526 def reverse(self):
1527 """Reverse the ordering of the QuerySet."""
1528 if self.query.is_sliced:
1529 raise TypeError("Cannot reverse a query once a slice has been taken.")
1530 clone = self._chain()
1531 clone.query.standard_ordering = not clone.query.standard_ordering
1532 return clone
1533
1534 def defer(self, *fields):
1535 """
1536 Defer the loading of data for certain fields until they are accessed.
1537 Add the set of deferred fields to any existing set of deferred fields.
1538 The only exception to this is if None is passed in as the only
1539 parameter, in which case removal all deferrals.
1540 """
1541 self._not_support_combined_queries("defer")
1542 if self._fields is not None:
1543 raise TypeError("Cannot call defer() after .values() or .values_list()")
1544 clone = self._chain()
1545 if fields == (None,):
1546 clone.query.clear_deferred_loading()
1547 else:
1548 clone.query.add_deferred_loading(fields)
1549 return clone
1550
1551 def only(self, *fields):
1552 """
1553 Essentially, the opposite of defer(). Only the fields passed into this
1554 method and that are not already specified as deferred are loaded
1555 immediately when the queryset is evaluated.
1556 """
1557 self._not_support_combined_queries("only")
1558 if self._fields is not None:
1559 raise TypeError("Cannot call only() after .values() or .values_list()")
1560 if fields == (None,):
1561 # Can only pass None to defer(), not only(), as the rest option.
1562 # That won't stop people trying to do this, so let's be explicit.
1563 raise TypeError("Cannot pass None as an argument to only().")
1564 for field in fields:
1565 field = field.split(LOOKUP_SEP, 1)[0]
1566 if field in self.query._filtered_relations:
1567 raise ValueError("only() is not supported with FilteredRelation.")
1568 clone = self._chain()
1569 clone.query.add_immediate_loading(fields)
1570 return clone
1571
1572 def using(self, alias):
1573 """Select which database this QuerySet should execute against."""
1574 clone = self._chain()
1575 clone._db = alias
1576 return clone
1577
1578 ###################################
1579 # PUBLIC INTROSPECTION ATTRIBUTES #
1580 ###################################
1581
1582 @property
1583 def ordered(self):
1584 """
1585 Return True if the QuerySet is ordered -- i.e. has an order_by()
1586 clause or a default ordering on the model (or is empty).
1587 """
1588 if isinstance(self, EmptyQuerySet):
1589 return True
1590 if self.query.extra_order_by or self.query.order_by:
1591 return True
1592 elif (
1593 self.query.default_ordering
1594 and self.query.get_meta().ordering
1595 and
1596 # A default ordering doesn't affect GROUP BY queries.
1597 not self.query.group_by
1598 ):
1599 return True
1600 else:
1601 return False
1602
1603 @property
1604 def db(self):
1605 """Return the database used if this query is executed now."""
1606 if self._for_write:
1607 return self._db or router.db_for_write(self.model, **self._hints)
1608 return self._db or router.db_for_read(self.model, **self._hints)
1609
1610 ###################
1611 # PRIVATE METHODS #
1612 ###################
1613
1614 def _insert(
1615 self,
1616 objs,
1617 fields,
1618 returning_fields=None,
1619 raw=False,
1620 using=None,
1621 on_conflict=None,
1622 update_fields=None,
1623 unique_fields=None,
1624 ):
1625 """
1626 Insert a new record for the given model. This provides an interface to
1627 the InsertQuery class and is how Model.save() is implemented.
1628 """
1629 self._for_write = True
1630 if using is None:
1631 using = self.db
1632 query = sql.InsertQuery(
1633 self.model,
1634 on_conflict=on_conflict,
1635 update_fields=update_fields,
1636 unique_fields=unique_fields,
1637 )
1638 query.insert_values(fields, objs, raw=raw)
1639 return query.get_compiler(using=using).execute_sql(returning_fields)
1640
1641 _insert.queryset_only = False
1642
1643 def _batched_insert(
1644 self,
1645 objs,
1646 fields,
1647 batch_size,
1648 on_conflict=None,
1649 update_fields=None,
1650 unique_fields=None,
1651 ):
1652 """
1653 Helper method for bulk_create() to insert objs one batch at a time.
1654 """
1655 connection = connections[self.db]
1656 ops = connection.ops
1657 max_batch_size = max(ops.bulk_batch_size(fields, objs), 1)
1658 batch_size = min(batch_size, max_batch_size) if batch_size else max_batch_size
1659 inserted_rows = []
1660 bulk_return = connection.features.can_return_rows_from_bulk_insert
1661 for item in [objs[i : i + batch_size] for i in range(0, len(objs), batch_size)]:
1662 if bulk_return and on_conflict is None:
1663 inserted_rows.extend(
1664 self._insert(
1665 item,
1666 fields=fields,
1667 using=self.db,
1668 returning_fields=self.model._meta.db_returning_fields,
1669 )
1670 )
1671 else:
1672 self._insert(
1673 item,
1674 fields=fields,
1675 using=self.db,
1676 on_conflict=on_conflict,
1677 update_fields=update_fields,
1678 unique_fields=unique_fields,
1679 )
1680 return inserted_rows
1681
1682 def _chain(self):
1683 """
1684 Return a copy of the current QuerySet that's ready for another
1685 operation.
1686 """
1687 obj = self._clone()
1688 if obj._sticky_filter:
1689 obj.query.filter_is_sticky = True
1690 obj._sticky_filter = False
1691 return obj
1692
1693 def _clone(self):
1694 """
1695 Return a copy of the current QuerySet. A lightweight alternative
1696 to deepcopy().
1697 """
1698 c = self.__class__(
1699 model=self.model,
1700 query=self.query.chain(),
1701 using=self._db,
1702 hints=self._hints,
1703 )
1704 c._sticky_filter = self._sticky_filter
1705 c._for_write = self._for_write
1706 c._prefetch_related_lookups = self._prefetch_related_lookups[:]
1707 c._known_related_objects = self._known_related_objects
1708 c._iterable_class = self._iterable_class
1709 c._fields = self._fields
1710 return c
1711
1712 def _fetch_all(self):
1713 if self._result_cache is None:
1714 self._result_cache = list(self._iterable_class(self))
1715 if self._prefetch_related_lookups and not self._prefetch_done:
1716 self._prefetch_related_objects()
1717
1718 def _next_is_sticky(self):
1719 """
1720 Indicate that the next filter call and the one following that should
1721 be treated as a single filter. This is only important when it comes to
1722 determining when to reuse tables for many-to-many filters. Required so
1723 that we can filter naturally on the results of related managers.
1724
1725 This doesn't return a clone of the current QuerySet (it returns
1726 "self"). The method is only used internally and should be immediately
1727 followed by a filter() that does create a clone.
1728 """
1729 self._sticky_filter = True
1730 return self
1731
1732 def _merge_sanity_check(self, other):
1733 """Check that two QuerySet classes may be merged."""
1734 if self._fields is not None and (
1735 set(self.query.values_select) != set(other.query.values_select)
1736 or set(self.query.extra_select) != set(other.query.extra_select)
1737 or set(self.query.annotation_select) != set(other.query.annotation_select)
1738 ):
1739 raise TypeError(
1740 f"Merging '{self.__class__.__name__}' classes must involve the same values in each case."
1741 )
1742
1743 def _merge_known_related_objects(self, other):
1744 """
1745 Keep track of all known related objects from either QuerySet instance.
1746 """
1747 for field, objects in other._known_related_objects.items():
1748 self._known_related_objects.setdefault(field, {}).update(objects)
1749
1750 def resolve_expression(self, *args, **kwargs):
1751 if self._fields and len(self._fields) > 1:
1752 # values() queryset can only be used as nested queries
1753 # if they are set up to select only a single field.
1754 raise TypeError("Cannot use multi-field values as a filter value.")
1755 query = self.query.resolve_expression(*args, **kwargs)
1756 query._db = self._db
1757 return query
1758
1759 resolve_expression.queryset_only = True
1760
1761 def _add_hints(self, **hints):
1762 """
1763 Update hinting information for use by routers. Add new key/values or
1764 overwrite existing key/values.
1765 """
1766 self._hints.update(hints)
1767
1768 def _has_filters(self):
1769 """
1770 Check if this QuerySet has any filtering going on. This isn't
1771 equivalent with checking if all objects are present in results, for
1772 example, qs[1:]._has_filters() -> False.
1773 """
1774 return self.query.has_filters()
1775
1776 @staticmethod
1777 def _validate_values_are_expressions(values, method_name):
1778 invalid_args = sorted(
1779 str(arg) for arg in values if not hasattr(arg, "resolve_expression")
1780 )
1781 if invalid_args:
1782 raise TypeError(
1783 "QuerySet.{}() received non-expression(s): {}.".format(
1784 method_name,
1785 ", ".join(invalid_args),
1786 )
1787 )
1788
1789 def _not_support_combined_queries(self, operation_name):
1790 if self.query.combinator:
1791 raise NotSupportedError(
1792 f"Calling QuerySet.{operation_name}() after {self.query.combinator}() is not supported."
1793 )
1794
1795 def _check_operator_queryset(self, other, operator_):
1796 if self.query.combinator or other.query.combinator:
1797 raise TypeError(f"Cannot use {operator_} operator with combined queryset.")
1798
1799 def _check_ordering_first_last_queryset_aggregation(self, method):
1800 if isinstance(self.query.group_by, tuple) and not any(
1801 col.output_field is self.model._meta.pk for col in self.query.group_by
1802 ):
1803 raise TypeError(
1804 f"Cannot use QuerySet.{method}() on an unordered queryset performing "
1805 f"aggregation. Add an ordering with order_by()."
1806 )
1807
1808
1809class InstanceCheckMeta(type):
1810 def __instancecheck__(self, instance):
1811 return isinstance(instance, QuerySet) and instance.query.is_empty()
1812
1813
1814class EmptyQuerySet(metaclass=InstanceCheckMeta):
1815 """
1816 Marker class to checking if a queryset is empty by .none():
1817 isinstance(qs.none(), EmptyQuerySet) -> True
1818 """
1819
1820 def __init__(self, *args, **kwargs):
1821 raise TypeError("EmptyQuerySet can't be instantiated")
1822
1823
1824class RawQuerySet:
1825 """
1826 Provide an iterator which converts the results of raw SQL queries into
1827 annotated model instances.
1828 """
1829
1830 def __init__(
1831 self,
1832 raw_query,
1833 model=None,
1834 query=None,
1835 params=(),
1836 translations=None,
1837 using=None,
1838 hints=None,
1839 ):
1840 self.raw_query = raw_query
1841 self.model = model
1842 self._db = using
1843 self._hints = hints or {}
1844 self.query = query or sql.RawQuery(sql=raw_query, using=self.db, params=params)
1845 self.params = params
1846 self.translations = translations or {}
1847 self._result_cache = None
1848 self._prefetch_related_lookups = ()
1849 self._prefetch_done = False
1850
1851 def resolve_model_init_order(self):
1852 """Resolve the init field names and value positions."""
1853 converter = connections[self.db].introspection.identifier_converter
1854 model_init_fields = [
1855 f for f in self.model._meta.fields if converter(f.column) in self.columns
1856 ]
1857 annotation_fields = [
1858 (column, pos)
1859 for pos, column in enumerate(self.columns)
1860 if column not in self.model_fields
1861 ]
1862 model_init_order = [
1863 self.columns.index(converter(f.column)) for f in model_init_fields
1864 ]
1865 model_init_names = [f.attname for f in model_init_fields]
1866 return model_init_names, model_init_order, annotation_fields
1867
1868 def prefetch_related(self, *lookups):
1869 """Same as QuerySet.prefetch_related()"""
1870 clone = self._clone()
1871 if lookups == (None,):
1872 clone._prefetch_related_lookups = ()
1873 else:
1874 clone._prefetch_related_lookups = clone._prefetch_related_lookups + lookups
1875 return clone
1876
1877 def _prefetch_related_objects(self):
1878 prefetch_related_objects(self._result_cache, *self._prefetch_related_lookups)
1879 self._prefetch_done = True
1880
1881 def _clone(self):
1882 """Same as QuerySet._clone()"""
1883 c = self.__class__(
1884 self.raw_query,
1885 model=self.model,
1886 query=self.query,
1887 params=self.params,
1888 translations=self.translations,
1889 using=self._db,
1890 hints=self._hints,
1891 )
1892 c._prefetch_related_lookups = self._prefetch_related_lookups[:]
1893 return c
1894
1895 def _fetch_all(self):
1896 if self._result_cache is None:
1897 self._result_cache = list(self.iterator())
1898 if self._prefetch_related_lookups and not self._prefetch_done:
1899 self._prefetch_related_objects()
1900
1901 def __len__(self):
1902 self._fetch_all()
1903 return len(self._result_cache)
1904
1905 def __bool__(self):
1906 self._fetch_all()
1907 return bool(self._result_cache)
1908
1909 def __iter__(self):
1910 self._fetch_all()
1911 return iter(self._result_cache)
1912
1913 def iterator(self):
1914 yield from RawModelIterable(self)
1915
1916 def __repr__(self):
1917 return f"<{self.__class__.__name__}: {self.query}>"
1918
1919 def __getitem__(self, k):
1920 return list(self)[k]
1921
1922 @property
1923 def db(self):
1924 """Return the database used if this query is executed now."""
1925 return self._db or router.db_for_read(self.model, **self._hints)
1926
1927 def using(self, alias):
1928 """Select the database this RawQuerySet should execute against."""
1929 return RawQuerySet(
1930 self.raw_query,
1931 model=self.model,
1932 query=self.query.chain(using=alias),
1933 params=self.params,
1934 translations=self.translations,
1935 using=alias,
1936 )
1937
1938 @cached_property
1939 def columns(self):
1940 """
1941 A list of model field names in the order they'll appear in the
1942 query results.
1943 """
1944 columns = self.query.get_columns()
1945 # Adjust any column names which don't match field names
1946 for query_name, model_name in self.translations.items():
1947 # Ignore translations for nonexistent column names
1948 try:
1949 index = columns.index(query_name)
1950 except ValueError:
1951 pass
1952 else:
1953 columns[index] = model_name
1954 return columns
1955
1956 @cached_property
1957 def model_fields(self):
1958 """A dict mapping column names to model field names."""
1959 converter = connections[self.db].introspection.identifier_converter
1960 model_fields = {}
1961 for field in self.model._meta.fields:
1962 name, column = field.get_attname_column()
1963 model_fields[converter(column)] = field
1964 return model_fields
1965
1966
1967class Prefetch:
1968 def __init__(self, lookup, queryset=None, to_attr=None):
1969 # `prefetch_through` is the path we traverse to perform the prefetch.
1970 self.prefetch_through = lookup
1971 # `prefetch_to` is the path to the attribute that stores the result.
1972 self.prefetch_to = lookup
1973 if queryset is not None and (
1974 isinstance(queryset, RawQuerySet)
1975 or (
1976 hasattr(queryset, "_iterable_class")
1977 and not issubclass(queryset._iterable_class, ModelIterable)
1978 )
1979 ):
1980 raise ValueError(
1981 "Prefetch querysets cannot use raw(), values(), and values_list()."
1982 )
1983 if to_attr:
1984 self.prefetch_to = LOOKUP_SEP.join(
1985 lookup.split(LOOKUP_SEP)[:-1] + [to_attr]
1986 )
1987
1988 self.queryset = queryset
1989 self.to_attr = to_attr
1990
1991 def __getstate__(self):
1992 obj_dict = self.__dict__.copy()
1993 if self.queryset is not None:
1994 queryset = self.queryset._chain()
1995 # Prevent the QuerySet from being evaluated
1996 queryset._result_cache = []
1997 queryset._prefetch_done = True
1998 obj_dict["queryset"] = queryset
1999 return obj_dict
2000
2001 def add_prefix(self, prefix):
2002 self.prefetch_through = prefix + LOOKUP_SEP + self.prefetch_through
2003 self.prefetch_to = prefix + LOOKUP_SEP + self.prefetch_to
2004
2005 def get_current_prefetch_to(self, level):
2006 return LOOKUP_SEP.join(self.prefetch_to.split(LOOKUP_SEP)[: level + 1])
2007
2008 def get_current_to_attr(self, level):
2009 parts = self.prefetch_to.split(LOOKUP_SEP)
2010 to_attr = parts[level]
2011 as_attr = self.to_attr and level == len(parts) - 1
2012 return to_attr, as_attr
2013
2014 def get_current_queryset(self, level):
2015 if self.get_current_prefetch_to(level) == self.prefetch_to:
2016 return self.queryset
2017 return None
2018
2019 def __eq__(self, other):
2020 if not isinstance(other, Prefetch):
2021 return NotImplemented
2022 return self.prefetch_to == other.prefetch_to
2023
2024 def __hash__(self):
2025 return hash((self.__class__, self.prefetch_to))
2026
2027
2028def normalize_prefetch_lookups(lookups, prefix=None):
2029 """Normalize lookups into Prefetch objects."""
2030 ret = []
2031 for lookup in lookups:
2032 if not isinstance(lookup, Prefetch):
2033 lookup = Prefetch(lookup)
2034 if prefix:
2035 lookup.add_prefix(prefix)
2036 ret.append(lookup)
2037 return ret
2038
2039
2040def prefetch_related_objects(model_instances, *related_lookups):
2041 """
2042 Populate prefetched object caches for a list of model instances based on
2043 the lookups/Prefetch instances given.
2044 """
2045 if not model_instances:
2046 return # nothing to do
2047
2048 # We need to be able to dynamically add to the list of prefetch_related
2049 # lookups that we look up (see below). So we need some book keeping to
2050 # ensure we don't do duplicate work.
2051 done_queries = {} # dictionary of things like 'foo__bar': [results]
2052
2053 auto_lookups = set() # we add to this as we go through.
2054 followed_descriptors = set() # recursion protection
2055
2056 all_lookups = normalize_prefetch_lookups(reversed(related_lookups))
2057 while all_lookups:
2058 lookup = all_lookups.pop()
2059 if lookup.prefetch_to in done_queries:
2060 if lookup.queryset is not None:
2061 raise ValueError(
2062 f"'{lookup.prefetch_to}' lookup was already seen with a different queryset. "
2063 "You may need to adjust the ordering of your lookups."
2064 )
2065
2066 continue
2067
2068 # Top level, the list of objects to decorate is the result cache
2069 # from the primary QuerySet. It won't be for deeper levels.
2070 obj_list = model_instances
2071
2072 through_attrs = lookup.prefetch_through.split(LOOKUP_SEP)
2073 for level, through_attr in enumerate(through_attrs):
2074 # Prepare main instances
2075 if not obj_list:
2076 break
2077
2078 prefetch_to = lookup.get_current_prefetch_to(level)
2079 if prefetch_to in done_queries:
2080 # Skip any prefetching, and any object preparation
2081 obj_list = done_queries[prefetch_to]
2082 continue
2083
2084 # Prepare objects:
2085 good_objects = True
2086 for obj in obj_list:
2087 # Since prefetching can re-use instances, it is possible to have
2088 # the same instance multiple times in obj_list, so obj might
2089 # already be prepared.
2090 if not hasattr(obj, "_prefetched_objects_cache"):
2091 try:
2092 obj._prefetched_objects_cache = {}
2093 except (AttributeError, TypeError):
2094 # Must be an immutable object from
2095 # values_list(flat=True), for example (TypeError) or
2096 # a QuerySet subclass that isn't returning Model
2097 # instances (AttributeError), either in Plain or a 3rd
2098 # party. prefetch_related() doesn't make sense, so quit.
2099 good_objects = False
2100 break
2101 if not good_objects:
2102 break
2103
2104 # Descend down tree
2105
2106 # We assume that objects retrieved are homogeneous (which is the premise
2107 # of prefetch_related), so what applies to first object applies to all.
2108 first_obj = obj_list[0]
2109 to_attr = lookup.get_current_to_attr(level)[0]
2110 prefetcher, descriptor, attr_found, is_fetched = get_prefetcher(
2111 first_obj, through_attr, to_attr
2112 )
2113
2114 if not attr_found:
2115 raise AttributeError(
2116 f"Cannot find '{through_attr}' on {first_obj.__class__.__name__} object, '{lookup.prefetch_through}' is an invalid "
2117 "parameter to prefetch_related()"
2118 )
2119
2120 if level == len(through_attrs) - 1 and prefetcher is None:
2121 # Last one, this *must* resolve to something that supports
2122 # prefetching, otherwise there is no point adding it and the
2123 # developer asking for it has made a mistake.
2124 raise ValueError(
2125 f"'{lookup.prefetch_through}' does not resolve to an item that supports "
2126 "prefetching - this is an invalid parameter to "
2127 "prefetch_related()."
2128 )
2129
2130 obj_to_fetch = None
2131 if prefetcher is not None:
2132 obj_to_fetch = [obj for obj in obj_list if not is_fetched(obj)]
2133
2134 if obj_to_fetch:
2135 obj_list, additional_lookups = prefetch_one_level(
2136 obj_to_fetch,
2137 prefetcher,
2138 lookup,
2139 level,
2140 )
2141 # We need to ensure we don't keep adding lookups from the
2142 # same relationships to stop infinite recursion. So, if we
2143 # are already on an automatically added lookup, don't add
2144 # the new lookups from relationships we've seen already.
2145 if not (
2146 prefetch_to in done_queries
2147 and lookup in auto_lookups
2148 and descriptor in followed_descriptors
2149 ):
2150 done_queries[prefetch_to] = obj_list
2151 new_lookups = normalize_prefetch_lookups(
2152 reversed(additional_lookups), prefetch_to
2153 )
2154 auto_lookups.update(new_lookups)
2155 all_lookups.extend(new_lookups)
2156 followed_descriptors.add(descriptor)
2157 else:
2158 # Either a singly related object that has already been fetched
2159 # (e.g. via select_related), or hopefully some other property
2160 # that doesn't support prefetching but needs to be traversed.
2161
2162 # We replace the current list of parent objects with the list
2163 # of related objects, filtering out empty or missing values so
2164 # that we can continue with nullable or reverse relations.
2165 new_obj_list = []
2166 for obj in obj_list:
2167 if through_attr in getattr(obj, "_prefetched_objects_cache", ()):
2168 # If related objects have been prefetched, use the
2169 # cache rather than the object's through_attr.
2170 new_obj = list(obj._prefetched_objects_cache.get(through_attr))
2171 else:
2172 try:
2173 new_obj = getattr(obj, through_attr)
2174 except exceptions.ObjectDoesNotExist:
2175 continue
2176 if new_obj is None:
2177 continue
2178 # We special-case `list` rather than something more generic
2179 # like `Iterable` because we don't want to accidentally match
2180 # user models that define __iter__.
2181 if isinstance(new_obj, list):
2182 new_obj_list.extend(new_obj)
2183 else:
2184 new_obj_list.append(new_obj)
2185 obj_list = new_obj_list
2186
2187
2188def get_prefetcher(instance, through_attr, to_attr):
2189 """
2190 For the attribute 'through_attr' on the given instance, find
2191 an object that has a get_prefetch_queryset().
2192 Return a 4 tuple containing:
2193 (the object with get_prefetch_queryset (or None),
2194 the descriptor object representing this relationship (or None),
2195 a boolean that is False if the attribute was not found at all,
2196 a function that takes an instance and returns a boolean that is True if
2197 the attribute has already been fetched for that instance)
2198 """
2199
2200 def has_to_attr_attribute(instance):
2201 return hasattr(instance, to_attr)
2202
2203 prefetcher = None
2204 is_fetched = has_to_attr_attribute
2205
2206 # For singly related objects, we have to avoid getting the attribute
2207 # from the object, as this will trigger the query. So we first try
2208 # on the class, in order to get the descriptor object.
2209 rel_obj_descriptor = getattr(instance.__class__, through_attr, None)
2210 if rel_obj_descriptor is None:
2211 attr_found = hasattr(instance, through_attr)
2212 else:
2213 attr_found = True
2214 if rel_obj_descriptor:
2215 # singly related object, descriptor object has the
2216 # get_prefetch_queryset() method.
2217 if hasattr(rel_obj_descriptor, "get_prefetch_queryset"):
2218 prefetcher = rel_obj_descriptor
2219 is_fetched = rel_obj_descriptor.is_cached
2220 else:
2221 # descriptor doesn't support prefetching, so we go ahead and get
2222 # the attribute on the instance rather than the class to
2223 # support many related managers
2224 rel_obj = getattr(instance, through_attr)
2225 if hasattr(rel_obj, "get_prefetch_queryset"):
2226 prefetcher = rel_obj
2227 if through_attr != to_attr:
2228 # Special case cached_property instances because hasattr
2229 # triggers attribute computation and assignment.
2230 if isinstance(
2231 getattr(instance.__class__, to_attr, None), cached_property
2232 ):
2233
2234 def has_cached_property(instance):
2235 return to_attr in instance.__dict__
2236
2237 is_fetched = has_cached_property
2238 else:
2239
2240 def in_prefetched_cache(instance):
2241 return through_attr in instance._prefetched_objects_cache
2242
2243 is_fetched = in_prefetched_cache
2244 return prefetcher, rel_obj_descriptor, attr_found, is_fetched
2245
2246
2247def prefetch_one_level(instances, prefetcher, lookup, level):
2248 """
2249 Helper function for prefetch_related_objects().
2250
2251 Run prefetches on all instances using the prefetcher object,
2252 assigning results to relevant caches in instance.
2253
2254 Return the prefetched objects along with any additional prefetches that
2255 must be done due to prefetch_related lookups found from default managers.
2256 """
2257 # prefetcher must have a method get_prefetch_queryset() which takes a list
2258 # of instances, and returns a tuple:
2259
2260 # (queryset of instances of self.model that are related to passed in instances,
2261 # callable that gets value to be matched for returned instances,
2262 # callable that gets value to be matched for passed in instances,
2263 # boolean that is True for singly related objects,
2264 # cache or field name to assign to,
2265 # boolean that is True when the previous argument is a cache name vs a field name).
2266
2267 # The 'values to be matched' must be hashable as they will be used
2268 # in a dictionary.
2269
2270 (
2271 rel_qs,
2272 rel_obj_attr,
2273 instance_attr,
2274 single,
2275 cache_name,
2276 is_descriptor,
2277 ) = prefetcher.get_prefetch_queryset(instances, lookup.get_current_queryset(level))
2278 # We have to handle the possibility that the QuerySet we just got back
2279 # contains some prefetch_related lookups. We don't want to trigger the
2280 # prefetch_related functionality by evaluating the query. Rather, we need
2281 # to merge in the prefetch_related lookups.
2282 # Copy the lookups in case it is a Prefetch object which could be reused
2283 # later (happens in nested prefetch_related).
2284 additional_lookups = [
2285 copy.copy(additional_lookup)
2286 for additional_lookup in getattr(rel_qs, "_prefetch_related_lookups", ())
2287 ]
2288 if additional_lookups:
2289 # Don't need to clone because the manager should have given us a fresh
2290 # instance, so we access an internal instead of using public interface
2291 # for performance reasons.
2292 rel_qs._prefetch_related_lookups = ()
2293
2294 all_related_objects = list(rel_qs)
2295
2296 rel_obj_cache = {}
2297 for rel_obj in all_related_objects:
2298 rel_attr_val = rel_obj_attr(rel_obj)
2299 rel_obj_cache.setdefault(rel_attr_val, []).append(rel_obj)
2300
2301 to_attr, as_attr = lookup.get_current_to_attr(level)
2302 # Make sure `to_attr` does not conflict with a field.
2303 if as_attr and instances:
2304 # We assume that objects retrieved are homogeneous (which is the premise
2305 # of prefetch_related), so what applies to first object applies to all.
2306 model = instances[0].__class__
2307 try:
2308 model._meta.get_field(to_attr)
2309 except exceptions.FieldDoesNotExist:
2310 pass
2311 else:
2312 msg = "to_attr={} conflicts with a field on the {} model."
2313 raise ValueError(msg.format(to_attr, model.__name__))
2314
2315 # Whether or not we're prefetching the last part of the lookup.
2316 leaf = len(lookup.prefetch_through.split(LOOKUP_SEP)) - 1 == level
2317
2318 for obj in instances:
2319 instance_attr_val = instance_attr(obj)
2320 vals = rel_obj_cache.get(instance_attr_val, [])
2321
2322 if single:
2323 val = vals[0] if vals else None
2324 if as_attr:
2325 # A to_attr has been given for the prefetch.
2326 setattr(obj, to_attr, val)
2327 elif is_descriptor:
2328 # cache_name points to a field name in obj.
2329 # This field is a descriptor for a related object.
2330 setattr(obj, cache_name, val)
2331 else:
2332 # No to_attr has been given for this prefetch operation and the
2333 # cache_name does not point to a descriptor. Store the value of
2334 # the field in the object's field cache.
2335 obj._state.fields_cache[cache_name] = val
2336 else:
2337 if as_attr:
2338 setattr(obj, to_attr, vals)
2339 else:
2340 manager = getattr(obj, to_attr)
2341 if leaf and lookup.queryset is not None:
2342 qs = manager._apply_rel_filters(lookup.queryset)
2343 else:
2344 qs = manager.get_queryset()
2345 qs._result_cache = vals
2346 # We don't want the individual qs doing prefetch_related now,
2347 # since we have merged this into the current work.
2348 qs._prefetch_done = True
2349 obj._prefetched_objects_cache[cache_name] = qs
2350 return all_related_objects, additional_lookups
2351
2352
2353class RelatedPopulator:
2354 """
2355 RelatedPopulator is used for select_related() object instantiation.
2356
2357 The idea is that each select_related() model will be populated by a
2358 different RelatedPopulator instance. The RelatedPopulator instances get
2359 klass_info and select (computed in SQLCompiler) plus the used db as
2360 input for initialization. That data is used to compute which columns
2361 to use, how to instantiate the model, and how to populate the links
2362 between the objects.
2363
2364 The actual creation of the objects is done in populate() method. This
2365 method gets row and from_obj as input and populates the select_related()
2366 model instance.
2367 """
2368
2369 def __init__(self, klass_info, select, db):
2370 self.db = db
2371 # Pre-compute needed attributes. The attributes are:
2372 # - model_cls: the possibly deferred model class to instantiate
2373 # - either:
2374 # - cols_start, cols_end: usually the columns in the row are
2375 # in the same order model_cls.__init__ expects them, so we
2376 # can instantiate by model_cls(*row[cols_start:cols_end])
2377 # - reorder_for_init: When select_related descends to a child
2378 # class, then we want to reuse the already selected parent
2379 # data. However, in this case the parent data isn't necessarily
2380 # in the same order that Model.__init__ expects it to be, so
2381 # we have to reorder the parent data. The reorder_for_init
2382 # attribute contains a function used to reorder the field data
2383 # in the order __init__ expects it.
2384 # - pk_idx: the index of the primary key field in the reordered
2385 # model data. Used to check if a related object exists at all.
2386 # - init_list: the field attnames fetched from the database. For
2387 # deferred models this isn't the same as all attnames of the
2388 # model's fields.
2389 # - related_populators: a list of RelatedPopulator instances if
2390 # select_related() descends to related models from this model.
2391 # - local_setter, remote_setter: Methods to set cached values on
2392 # the object being populated and on the remote object. Usually
2393 # these are Field.set_cached_value() methods.
2394 select_fields = klass_info["select_fields"]
2395
2396 self.cols_start = select_fields[0]
2397 self.cols_end = select_fields[-1] + 1
2398 self.init_list = [
2399 f[0].target.attname for f in select[self.cols_start : self.cols_end]
2400 ]
2401 self.reorder_for_init = None
2402
2403 self.model_cls = klass_info["model"]
2404 self.pk_idx = self.init_list.index(self.model_cls._meta.pk.attname)
2405 self.related_populators = get_related_populators(klass_info, select, self.db)
2406 self.local_setter = klass_info["local_setter"]
2407 self.remote_setter = klass_info["remote_setter"]
2408
2409 def populate(self, row, from_obj):
2410 if self.reorder_for_init:
2411 obj_data = self.reorder_for_init(row)
2412 else:
2413 obj_data = row[self.cols_start : self.cols_end]
2414 if obj_data[self.pk_idx] is None:
2415 obj = None
2416 else:
2417 obj = self.model_cls.from_db(self.db, self.init_list, obj_data)
2418 for rel_iter in self.related_populators:
2419 rel_iter.populate(row, obj)
2420 self.local_setter(from_obj, obj)
2421 if obj is not None:
2422 self.remote_setter(obj, from_obj)
2423
2424
2425def get_related_populators(klass_info, select, db):
2426 iterators = []
2427 related_klass_infos = klass_info.get("related_klass_infos", [])
2428 for rel_klass_info in related_klass_infos:
2429 rel_cls = RelatedPopulator(rel_klass_info, select, db)
2430 iterators.append(rel_cls)
2431 return iterators