CAVaccineInventory / vial

Sentry Issue: VIAL-BK

InterfaceError: connection already closed
  File "django/db/backends/base/base.py", line 242, in _commit
    return self.connection.commit()

InterfaceError: connection already closed
(1 additional frame(s) were not displayed)
...
  File "django/utils/asyncio.py", line 26, in inner
    return func(*args, **kwargs)
  File "django/db/backends/base/base.py", line 266, in commit
    self._commit()
  File "django/db/backends/base/base.py", line 242, in _commit
    return self.connection.commit()
  File "django/db/utils.py", line 90, in __exit__
    raise dj_exc_value.with_traceback(traceback) from exc_value
  File "django/db/backends/base/base.py", line 242, in _commit
    return self.connection.commit()

InterfaceError: connection already closed
  File "django/db/backends/postgresql/base.py", line 277, in _set_autocommit
    self.connection.autocommit = autocommit

InterfaceError: connection already closed
(6 additional frame(s) were not displayed)
...
  File "django/db/transaction.py", line 290, in __exit__
    connection.set_autocommit(True)
  File "django/db/backends/base/base.py", line 415, in set_autocommit
    self._set_autocommit(autocommit)
  File "django/db/backends/postgresql/base.py", line 277, in _set_autocommit
    self.connection.autocommit = autocommit
  File "django/db/utils.py", line 90, in __exit__
    raise dj_exc_value.with_traceback(traceback) from exc_value
  File "django/db/backends/postgresql/base.py", line 277, in _set_autocommit
    self.connection.autocommit = autocommit

InterfaceError: connection already closed
  File "django/db/backends/base/base.py", line 237, in _cursor
    return self._prepare_cursor(self.create_cursor(name))
  File "django/utils/asyncio.py", line 26, in inner
    return func(*args, **kwargs)
  File "django/db/backends/postgresql/base.py", line 236, in create_cursor
    cursor = self.connection.cursor()

InterfaceError: connection already closed
(16 additional frame(s) were not displayed)
...
  File "django/db/backends/base/base.py", line 237, in _cursor
    return self._prepare_cursor(self.create_cursor(name))
  File "django/db/utils.py", line 90, in __exit__
    raise dj_exc_value.with_traceback(traceback) from exc_value
  File "django/db/backends/base/base.py", line 237, in _cursor
    return self._prepare_cursor(self.create_cursor(name))
  File "django/utils/asyncio.py", line 26, in inner
    return func(*args, **kwargs)
  File "django/db/backends/postgresql/base.py", line 236, in create_cursor
    cursor = self.connection.cursor()

From Sentry:

This makes me suspect that it's related to the change I just shipped from #705 - I added providers, and it looks like that might be an N+1 query against providers - which wasn't supposed to happen since I added a .select_related() here:

vial/vaccinate/api/serialize.py

Lines 217 to 236 in ffff6a1

    
           formats["v0preview"] = OutputFormat( 
        
               prepare_queryset=lambda qs: qs.select_related( 
        
                   "dn_latest_non_skip_report", "provider" 
        
               ), 
        
               start=( 
        
                   b'{"usage":{"notice":"Please contact Vaccinate The States and let ' 
        
                   b"us know if you plan to rely on or publish this data. This " 
        
                   b"data is provided with best-effort accuracy. If you are " 
        
                   b"displaying this data, we expect you to display it responsibly. " 
        
                   b'Please do not display it in a way that is easy to misread.",' 
        
                   b'"contact":{"partnersEmail":"api@vaccinatethestates.com"}},' 
        
                   b'"content":[' 
        
               ), 
        
               transform=lambda l: location_v0_json(l), 
        
               transform_batch=transform_batch, 
        
               serialize=orjson.dumps, 
        
               separator=b",", 
        
               end=lambda qs: b"]}", 
        
               content_type="application/json", 
        
           )

Debug toolbar confirms I messed this up:

The SQL query looks like this:

SELECT
  DISTINCT "location"."id",
  "location"."name",
  "location"."phone_number",
  "location"."full_address",
  "location"."city",
  "location"."state_id",
  "location"."zip_code",
  "location"."hours",
  "location"."website",
  "location"."location_type_id",
  "location"."google_places_id",
  "location"."vaccinespotter_location_id",
  "location"."vaccinefinder_location_id",
  "location"."provider_id",
  "location"."county_id",
  "location"."latitude",
  "location"."longitude",
  "location"."public_id",
  "location"."import_ref",
  "location"."preferred_contact_method",
  "location"."dn_latest_non_skip_report_id",
  "state"."id",
  "state"."abbreviation",
  "location_type"."id",
  "location_type"."name",
  "provider"."id",
  "provider"."name",
  "provider"."provider_type_id",
  "provider_type"."id",
  "provider_type"."name",
  "county"."id",
  "county"."name",
  "report"."id",
  "report"."location_id",
  "report"."is_pending_review",
  "report"."originally_pending_review",
  "report"."pending_review_because",
  "report"."claimed_by_id",
  "report"."claimed_at",
  "report"."soft_deleted",
  "report"."soft_deleted_because",
  "report"."report_source",
  "report"."appointment_tag_id",
  "report"."appointment_details",
  "report"."public_notes",
  "report"."internal_notes",
  "report"."restriction_notes",
  "report"."vaccines_offered",
  "report"."website",
  "report"."full_address",
  "report"."hours",
  "report"."planned_closure",
  "report"."reported_by_id",
  "report"."created_at",
  "report"."call_request_id",
  "report"."airtable_id",
  "report"."airtable_json",
  "report"."public_id"
FROM
  "location"
  INNER JOIN "state" ON ("location"."state_id" = "state"."id")
  INNER JOIN "location_type" ON (
    "location"."location_type_id" = "location_type"."id"
  )
  LEFT OUTER JOIN "provider" ON ("location"."provider_id" = "provider"."id")
  LEFT OUTER JOIN "provider_type" ON (
    "provider"."provider_type_id" = "provider_type"."id"
  )
  LEFT OUTER JOIN "county" ON ("location"."county_id" = "county"."id")
  LEFT OUTER JOIN "report" ON (
    "location"."dn_latest_non_skip_report_id" = "report"."id"
  )
WHERE
  NOT "location"."soft_deleted"

See how it selects these columns:

  "provider"."id",
  "provider"."name",
  "provider"."provider_type_id",

But NOT the vaccine_info_url column we use.

Here's why: it routes through location_json_queryset which does this:

vial/vaccinate/api/serialize.py

Lines 69 to 99 in b31b11f

    
           def location_json_queryset(queryset: QuerySet[Location]) -> QuerySet[Location]: 
        
               return ( 
        
                   queryset.select_related( 
        
                       "state", 
        
                       "county", 
        
                       "location_type", 
        
                       "provider__provider_type", 
        
                   ).prefetch_related("concordances") 
        
               ).only( 
        
                   "public_id", 
        
                   "name", 
        
                   "state__abbreviation", 
        
                   "latitude", 
        
                   "longitude", 
        
                   "location_type__name", 
        
                   "import_ref", 
        
                   "phone_number", 
        
                   "full_address", 
        
                   "city", 
        
                   "county__name", 
        
                   "google_places_id", 
        
                   "vaccinefinder_location_id", 
        
                   "vaccinespotter_location_id", 
        
                   "zip_code", 
        
                   "hours", 
        
                   "website", 
        
                   "preferred_contact_method", 
        
                   "provider__name", 
        
                   "provider__provider_type__name", 
        
                   "dn_latest_non_skip_report", 
        
               )

	formats["v0preview"] = OutputFormat(
	prepare_queryset=lambda qs: qs.select_related(
	"dn_latest_non_skip_report", "provider"
	),
	start=(
	b'{"usage":{"notice":"Please contact Vaccinate The States and let '
	b"us know if you plan to rely on or publish this data. This "
	b"data is provided with best-effort accuracy. If you are "
	b"displaying this data, we expect you to display it responsibly. "
	b'Please do not display it in a way that is easy to misread.",'
	b'"contact":{"partnersEmail":"api@vaccinatethestates.com"}},'
	b'"content":['
	),
	transform=lambda l: location_v0_json(l),
	transform_batch=transform_batch,
	serialize=orjson.dumps,
	separator=b",",
	end=lambda qs: b"]}",
	content_type="application/json",
	)

	def location_json_queryset(queryset: QuerySet[Location]) -> QuerySet[Location]:
	return (
	queryset.select_related(
	"state",
	"county",
	"location_type",
	"provider__provider_type",
	).prefetch_related("concordances")
	).only(
	"public_id",
	"name",
	"state__abbreviation",
	"latitude",
	"longitude",
	"location_type__name",
	"import_ref",
	"phone_number",
	"full_address",
	"city",
	"county__name",
	"google_places_id",
	"vaccinefinder_location_id",
	"vaccinespotter_location_id",
	"zip_code",
	"hours",
	"website",
	"preferred_contact_method",
	"provider__name",
	"provider__provider_type__name",
	"dn_latest_non_skip_report",
	)

/api/export: InterfaceError: connection already closed - caused by N+1 SQL queries