Skip to content

Commit

Permalink
update convert methods to support monolithic loading
Browse files Browse the repository at this point in the history
  • Loading branch information
@weaverba137
weaverba137 committed Sep 25, 2024
1 parent f5de544 commit 364bf34
Show file tree
Hide file tree
Showing 4 changed files with 383 additions and 575 deletions.
155 changes: 155 additions & 0 deletions py/specprodDB/attic/zpix_target.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
def zpix_target(specprod):
"""Replace targeting bitmasks in the redshift tables for `specprod`.

Parameters
----------
specprod : :class:`str`
The spectroscopic production, normally the value of :envvar:`SPECPROD`.
"""
specprod_survey_program = {'fuji': {'cmx': ('other', ),
'special': ('dark', ),
'sv1': ('backup', 'bright', 'dark', 'other'),
'sv2': ('backup', 'bright', 'dark'),
'sv3': ('backup', 'bright', 'dark')},
'guadalupe': {'special': ('bright', 'dark'),
'main': ('bright', 'dark')},
'iron': {'cmx': ('other', ),
'main': ('backup', 'bright', 'dark'),
'special': ('backup', 'bright', 'dark', 'other'),
'sv1': ('backup', 'bright', 'dark', 'other'),
'sv2': ('backup', 'bright', 'dark'),
'sv3': ('backup', 'bright', 'dark')}}
target_bits = {'cmx': {'cmx': Target.cmx_target},
'sv1': {'desi': Target.sv1_desi_target, 'bgs': Target.sv1_bgs_target, 'mws': Target.sv1_mws_target},
'sv2': {'desi': Target.sv2_desi_target, 'bgs': Target.sv2_bgs_target, 'mws': Target.sv2_mws_target},
'sv3': {'desi': Target.sv3_desi_target, 'bgs': Target.sv3_bgs_target, 'mws': Target.sv3_mws_target},
'main': {'desi': Target.desi_target, 'bgs': Target.bgs_target, 'mws': Target.mws_target},
'special': {'desi': Target.desi_target, 'bgs': Target.bgs_target, 'mws': Target.mws_target}}
#
# Find targetid assigned to multiple tiles.
#
assigned_multiple_tiles = dict()
for survey in specprod_survey_program[specprod]:
assigned_multiple_tiles[survey] = dict()
for program in specprod_survey_program[specprod][survey]:
assigned_multiple_tiles[survey][program] = dbSession.query(Target.targetid).join(Fiberassign,
and_(Target.targetid == Fiberassign.targetid,
Target.tileid == Fiberassign.tileid)).filter(Target.survey == survey).filter(Target.program == program).group_by(Target.targetid).having(func.count(Target.tileid) > 1)
#
# From that set, find cases targetid and a targeting bit are distinct pairs.
#
distinct_target = dict()
for survey in assigned_multiple_tiles:
distinct_target[survey] = dict()
for program in assigned_multiple_tiles[survey]:
distinct_target[survey][program] = dict()
for bits in target_bits[survey]:
distinct_target[survey][program][bits] = dbSession.query(Target.targetid, target_bits[survey][bits]).filter(Target.targetid.in_(assigned_multiple_tiles[survey][program])).filter(Target.survey == survey).filter(Target.program == program).distinct().subquery()
#
# Obtain the list of targetids where a targeting bit appears more than once with different values.
#
multiple_target = dict()
for survey in distinct_target:
multiple_target[survey] = dict()
for program in distinct_target[survey]:
multiple_target[survey][program] = dict()
for bits in distinct_target[survey][program]:
if survey.startswith('sv'):
column = getattr(distinct_target[survey][program][bits].c, f"{survey}_{bits}_target")
elif survey == 'cmx':
column = distinct_target[survey][program][bits].c.cmx_target
else:
column = getattr(distinct_target[survey][program][bits].c, f"{bits}_target")
multiple_target[survey][program][bits] = [row[0] for row in dbSession.query(distinct_target[survey][program][bits].c.targetid).group_by(distinct_target[survey][program][bits].c.targetid).having(func.count(column) > 1).all()]
#
# Consolidate the list of targetids.
#
targetids_to_fix = dict()
for survey in multiple_target:
for program in multiple_target[survey]:
for bits in multiple_target[survey][program]:
if multiple_target[survey][program][bits]:
if survey not in targetids_to_fix:
targetids_to_fix[survey] = dict()
if program in targetids_to_fix[survey]:
log.debug("targetids_to_fix['%s']['%s'] += multiple_target['%s']['%s']['%s']",
survey, program, survey, program, bits)
targetids_to_fix[survey][program] += multiple_target[survey][program][bits]
else:
log.debug("targetids_to_fix['%s']['%s'] = multiple_target['%s']['%s']['%s']",
survey, program, survey, program, bits)
targetids_to_fix[survey][program] = multiple_target[survey][program][bits]
#
# ToO observations that had targeting bits zeroed out.
#
if specprod == 'fuji':
#
# Maybe this problem only affects fuji, but need to confirm that.
#
zero_ToO = dict()
for survey in specprod_survey_program[specprod]:
zero_ToO[survey] = dict()
for program in specprod_survey_program[specprod][survey]:
zero_ToO[survey][program] = [row[0] for row in dbSession.query(Zpix.targetid).filter((Zpix.targetid.op('&')((2**16 - 1) << 42)).op('>>')(42) == 9999).filter(Zpix.survey == survey).filter(Zpix.program == program).all()]
for survey in zero_ToO:
for program in zero_ToO[survey]:
if zero_ToO[survey][program]:
if survey not in targetids_to_fix:
targetids_to_fix[survey] = dict()
if program in targetids_to_fix[survey]:
log.debug("targetids_to_fix['%s']['%s'] += zero_ToO['%s']['%s']",
survey, program, survey, program)
targetids_to_fix[survey][program] += zero_ToO[survey][program]
else:
log.debug("targetids_to_fix['%s']['%s'] = zero_ToO['%s']['%s']",
survey, program, survey, program)
targetids_to_fix[survey][program] = zero_ToO[survey][program]
#
# Generate the query to obtain the bitwise-or of each targeting bit.
#
# table = 'zpix'
surveys = ('', 'sv1', 'sv2', 'sv3')
programs = ('desi', 'bgs', 'mws', 'scnd')
masks = ['cmx_target'] + [('_'.join(p) if p[0] else p[1]) + '_target'
for p in itertools.product(surveys, programs)]
bit_or_query = dict()
for survey in targetids_to_fix:
bit_or_query[survey] = dict()
for program in targetids_to_fix[survey]:
log.debug("SELECT t.targetid, " +
', '.join([f"BIT_OR(t.{m}) AS {m}" for m in masks]) +
f" FROM {specprod}.target AS t WHERE t.targetid IN ({', '.join(map(str, set(targetids_to_fix[survey][program])))}) AND t.survey = '{survey}' AND t.program = '{program}' GROUP BY t.targetid;")
bq = ("dbSession.query(Target.targetid, " +
', '.join([f"func.bit_or(Target.{m}).label('{m}')" for m in masks]) +
f").filter(Target.targetid.in_([{', '.join(map(str, set(targetids_to_fix[survey][program])))}])).filter(Target.survey == '{survey}').filter(Target.program == '{program}').group_by(Target.targetid)")
log.debug(bq)
bit_or_query[survey][program] = eval(bq)
#
# Apply the updates
#
# update_string = '{' + ', '.join([f"Zpix.{m}: {{0.{m}:d}}" for m in masks]) + '}'
for survey in bit_or_query:
for program in bit_or_query[survey]:
for row in bit_or_query[survey][program].all():
zpix_match = dbSession.query(Zpix).filter(Zpix.targetid == row.targetid).filter(Zpix.survey == survey).filter(Zpix.program == program).one()
for m in masks:
log.info("%s.%s = %s", zpix_match, m, str(getattr(row, m)))
zpix_match.cmx_target = row.cmx_target
zpix_match.desi_target = row.desi_target
zpix_match.bgs_target = row.bgs_target
zpix_match.mws_target = row.mws_target
zpix_match.scnd_target = row.scnd_target
zpix_match.sv1_desi_target = row.sv1_desi_target
zpix_match.sv1_bgs_target = row.sv1_bgs_target
zpix_match.sv1_mws_target = row.sv1_mws_target
zpix_match.sv1_scnd_target = row.sv1_scnd_target
zpix_match.sv2_desi_target = row.sv2_desi_target
zpix_match.sv2_bgs_target = row.sv2_bgs_target
zpix_match.sv2_mws_target = row.sv2_mws_target
zpix_match.sv2_scnd_target = row.sv2_scnd_target
zpix_match.sv3_desi_target = row.sv3_desi_target
zpix_match.sv3_bgs_target = row.sv3_bgs_target
zpix_match.sv3_mws_target = row.sv3_mws_target
zpix_match.sv3_scnd_target = row.sv3_scnd_target
dbSession.commit()
return
4 changes: 2 additions & 2 deletions py/specprodDB/data/load_specprod_db.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
# This is intended to configure (meta)data releated to the load process itself.
#
username = desi_admin
hostname = specprod-db.desi.lbl.gov
# hostname = specprod-db.desi.lbl.gov
hostname = db2-loadbalancer.specprod.production.svc.spin.nersc.org
chunksize = 50000
maxrows = 0

[fuji]
#
Expand Down
Loading

0 comments on commit 364bf34

Please sign in to comment.