Skip to content

feat: scripts for commissioning run on November 2022 #20

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
monodera merged 1 commit into from
Dec 7, 2022
Merged

feat: scripts for commissioning run on November 2022 #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
179 changes: 179 additions & 0 deletions examples/commissioning_2022nov/calstar/insert_yamashita_stars.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,179 @@
#!/usr/bin/env python3

import argparse
import datetime

import numpy as np
import pandas as pd
import toml
from astropy.table import Table
from logzero import logger
from targetdb_api_utils import insert_simple
from targetdb_api_utils import join_backref_values
from targetdb_api_utils import update_simple

from targetdb import targetdb


def load_data(infile):
tb = Table.read(infile, format="ascii.ecsv")
proposal_id = tb.meta["proposal_id"]
df = tb.to_pandas()
print(df)
return df, proposal_id


def format_data(df, proposal_id):

filter_bands = ["g", "r", "i", "z", "y", "j"]

dfout = df.copy(deep=True)

dfout.rename(
columns={"input_catalog": "input_catalog_name"},
inplace=True,
)
dfout["proposal_id"] = proposal_id
dfout["target_type_name"] = "SCIENCE"

lists_filtername = {"g": [], "r": [], "i": [], "z": [], "y": [], "j": []}

for i in range(df.index.size):

filter_name = df["filter_name"][i]

if "APASS" in filter_name:
filter_system = "sdss"
elif "PS1" in filter_name:
filter_system = "ps1"
else:
logger.error(f"Filter system is not registered. Check the code.")
exit()

for band in filter_bands:
try:
if np.isfinite(dfout[f"psf_mag_{band}"][i]) and np.isfinite(
dfout[f"psf_flux_{band}"][i]
):
lists_filtername[band].append(f"{band}_{filter_system}")
else:
logger.warning(
f"NaN is found in the {band} flux and magnitude for the object {df['obj_id'][i]}. None is put there."
)
lists_filtername[band].append(None)
except KeyError as e:
logger.warning(
f"No {band} information is found in the data and the columns are set as NULL in the database."
)
lists_filtername[band].append(None)

for band in filter_bands:
dfout[f"filter_{band}"] = lists_filtername[band]

print(dfout)

return dfout


def main(conf, infile, dry_run=False):

logger.info(f"Loading input data into dataframe")
df, proposal_id = load_data(infile)

df = format_data(df, proposal_id)

logger.info(f"Load config file {conf}.")
config = toml.load(conf)

logger.info(f"Connect to the database.")
db = targetdb.TargetDB(**dict(config["targetdb"]["db"]))
db.connect()

backref_tables = ["proposal", "input_catalog", "target_type"]
backref_keys = ["proposal_id", "input_catalog_name", "target_type_name"]
backref_check_keys = ["proposal_id", "input_catalog_id", "target_type_id"]

for i in range(len(backref_tables)):
df = join_backref_values(
df,
db=db,
table=backref_tables[i],
key=backref_keys[i],
check_key=backref_check_keys[i],
)

logger.debug(df[["proposal_id", "input_catalog_name", "target_type_name"]])

for i in range(len(df)):
# logger.info(
# f"""
# {df.loc[i,]}"""
# )
query = f"""
SELECT target_id, obj_id
FROM target
WHERE obj_id = {df.loc[i, "obj_id"]} AND input_catalog_id = {df.loc[i, "input_catalog_id"]}
;
"""
df_target = db.fetch_query(query)
if df_target.empty:
# 3097940536010212736
# 3830980604624181376
# 892231562565363072
logger.info(f"""{df.loc[i, "obj_id"]} NOT found in the target table""")
df_update = df.iloc[[i]].copy(deep=True)
logger.info(f"""{df_update}""")
logger.info(f"{type(df_update)}")
if not dry_run:
logger.info(f"Insert data into the database.")
db = insert_simple(
db,
table="target",
df=df_update,
fetch_table=False,
)
else:
logger.info(f"""{df.loc[i,"obj_id"]} found in the target table""")
df_update = df.iloc[[i]].copy(deep=True)
df_update["target_id"] = [df_target.loc[0, "target_id"]]
logger.info(f"""{df_update}""")
logger.info(f"{type(df_update)}")
if not dry_run:
logger.info(f"Update data in the database.")
db = update_simple(
db,
table="target",
df=df_update,
fetch_table=False,
)

logger.info(f"Close the database.")
db.close()


if __name__ == "__main__":

parser = argparse.ArgumentParser()

parser.add_argument(
"conf",
type=str,
help="Config file for the script to run. Must be a .toml file (mandatory)",
)

parser.add_argument(
"--infile",
type=str,
default="../../../../external_data/commissioning_2022sep/stars_yamashita/targets_S22B-EN16.ecsv",
help="Input file from Yamashita-san",
)

parser.add_argument(
"--dry_run",
action="store_true",
help="Dry-run (no update of the database)",
)

args = parser.parse_args()

main(args.conf, args.infile, args.dry_run)
125 changes: 125 additions & 0 deletions examples/commissioning_2022nov/calstar/log.update
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
[I 221110 19:29:00 insert_yamashita_stars:80] Loading input data into dataframe
[W 221110 19:29:00 insert_yamashita_stars:65] No j information is found in the data and the columns are set as NULL in the database.
[W 221110 19:29:00 insert_yamashita_stars:65] No j information is found in the data and the columns are set as NULL in the database.
[W 221110 19:29:00 insert_yamashita_stars:65] No j information is found in the data and the columns are set as NULL in the database.
[W 221110 19:29:00 insert_yamashita_stars:65] No j information is found in the data and the columns are set as NULL in the database.
[W 221110 19:29:00 insert_yamashita_stars:60] NaN is found in the y flux and magnitude for the object 3097940536010212736. None is put there.
[W 221110 19:29:00 insert_yamashita_stars:65] No j information is found in the data and the columns are set as NULL in the database.
[W 221110 19:29:00 insert_yamashita_stars:65] No j information is found in the data and the columns are set as NULL in the database.
[W 221110 19:29:00 insert_yamashita_stars:65] No j information is found in the data and the columns are set as NULL in the database.
[W 221110 19:29:00 insert_yamashita_stars:65] No j information is found in the data and the columns are set as NULL in the database.
[W 221110 19:29:00 insert_yamashita_stars:65] No j information is found in the data and the columns are set as NULL in the database.
[W 221110 19:29:00 insert_yamashita_stars:60] NaN is found in the z flux and magnitude for the object 5057499161282786432. None is put there.
[W 221110 19:29:00 insert_yamashita_stars:60] NaN is found in the y flux and magnitude for the object 5057499161282786432. None is put there.
[W 221110 19:29:00 insert_yamashita_stars:65] No j information is found in the data and the columns are set as NULL in the database.
[W 221110 19:29:00 insert_yamashita_stars:60] NaN is found in the y flux and magnitude for the object 4881345510044485120. None is put there.
[W 221110 19:29:00 insert_yamashita_stars:65] No j information is found in the data and the columns are set as NULL in the database.
[I 221110 19:29:00 insert_yamashita_stars:85] Load config file /work/monodera/Subaru-PFS/database_configs/config_pfsa-db01-gb_commissioning_2022nov.toml.
[I 221110 19:29:00 insert_yamashita_stars:88] Connect to the database.
[D 221110 19:29:00 insert_yamashita_stars:105] proposal_id input_catalog_name target_type_name
0 S22B-EN16 gaia_dr3 SCIENCE
1 S22B-EN16 gaia_dr3 SCIENCE
2 S22B-EN16 gaia_dr3 SCIENCE
3 S22B-EN16 gaia_dr3 SCIENCE
4 S22B-EN16 gaia_dr3 SCIENCE
5 S22B-EN16 gaia_dr3 SCIENCE
6 S22B-EN16 gaia_dr3 SCIENCE
7 S22B-EN16 gaia_dr3 SCIENCE
8 S22B-EN16 gaia_dr3 SCIENCE
9 S22B-EN16 gaia_dr3 SCIENCE
10 S22B-EN16 gaia_dr3 SCIENCE
[I 221110 19:29:00 insert_yamashita_stars:136] 2536159496590552704 found in the target table
[I 221110 19:29:00 insert_yamashita_stars:139] obj_id ra ... updated_at target_id
0 2536159496590552704 15.842504 ... 2022-04-30 02:07:08.500763 77188

[1 rows x 58 columns]
[I 221110 19:29:00 insert_yamashita_stars:140] <class 'pandas.core.frame.DataFrame'>
[I 221110 19:29:00 insert_yamashita_stars:136] 5176546064064586624 found in the target table
[I 221110 19:29:00 insert_yamashita_stars:139] obj_id ra ... updated_at target_id
1 5176546064064586624 37.071596 ... 2022-04-30 02:07:08.500763 77189

[1 rows x 58 columns]
[I 221110 19:29:00 insert_yamashita_stars:140] <class 'pandas.core.frame.DataFrame'>
[I 221110 19:29:00 insert_yamashita_stars:136] 139724391470489472 found in the target table
[I 221110 19:29:00 insert_yamashita_stars:139] obj_id ra ... updated_at target_id
2 139724391470489472 42.229021 ... 2022-04-30 02:07:08.500763 77190

[1 rows x 58 columns]
[I 221110 19:29:00 insert_yamashita_stars:140] <class 'pandas.core.frame.DataFrame'>
[I 221110 19:29:00 insert_yamashita_stars:136] 892231562565363072 found in the target table
[I 221110 19:29:00 insert_yamashita_stars:139] obj_id ra ... updated_at target_id
3 892231562565363072 111.969802 ... 2022-04-30 02:07:08.500763 103970

[1 rows x 58 columns]
[I 221110 19:29:00 insert_yamashita_stars:140] <class 'pandas.core.frame.DataFrame'>
[I 221110 19:29:00 insert_yamashita_stars:136] 3097940536010212736 found in the target table
[I 221110 19:29:00 insert_yamashita_stars:139] obj_id ra ... updated_at target_id
4 3097940536010212736 123.786591 ... 2022-04-30 02:07:08.500763 103971

[1 rows x 58 columns]
[I 221110 19:29:00 insert_yamashita_stars:140] <class 'pandas.core.frame.DataFrame'>
[I 221110 19:29:01 insert_yamashita_stars:136] 3830980604624181376 found in the target table
[I 221110 19:29:01 insert_yamashita_stars:139] obj_id ra ... updated_at target_id
5 3830980604624181376 156.128799 ... 2022-04-30 02:07:08.500763 103972

[1 rows x 58 columns]
[I 221110 19:29:01 insert_yamashita_stars:140] <class 'pandas.core.frame.DataFrame'>
[I 221110 19:29:01 insert_yamashita_stars:136] 6910475935427725824 found in the target table
[I 221110 19:29:01 insert_yamashita_stars:139] obj_id ra ... updated_at target_id
6 6910475935427725824 315.461113 ... 2022-04-30 02:07:08.500763 77191

[1 rows x 58 columns]
[I 221110 19:29:01 insert_yamashita_stars:140] <class 'pandas.core.frame.DataFrame'>
[I 221110 19:29:01 insert_yamashita_stars:136] 2644572064644349952 found in the target table
[I 221110 19:29:01 insert_yamashita_stars:139] obj_id ra ... updated_at target_id
7 2644572064644349952 352.422169 ... 2022-04-30 02:07:08.500763 77192

[1 rows x 58 columns]
[I 221110 19:29:01 insert_yamashita_stars:140] <class 'pandas.core.frame.DataFrame'>
[I 221110 19:29:01 insert_yamashita_stars:136] 2881271732415859072 found in the target table
[I 221110 19:29:01 insert_yamashita_stars:139] obj_id ra ... updated_at target_id
8 2881271732415859072 357.934477 ... 2022-04-30 02:07:08.500763 77193

[1 rows x 58 columns]
[I 221110 19:29:01 insert_yamashita_stars:140] <class 'pandas.core.frame.DataFrame'>
[I 221110 19:29:01 insert_yamashita_stars:136] 5057499161282786432 found in the target table
[I 221110 19:29:01 insert_yamashita_stars:139] obj_id ra ... updated_at target_id
9 5057499161282786432 53.137007 ... 2022-04-30 02:07:08.500763 77186

[1 rows x 58 columns]
[I 221110 19:29:01 insert_yamashita_stars:140] <class 'pandas.core.frame.DataFrame'>
[I 221110 19:29:01 insert_yamashita_stars:136] 4881345510044485120 found in the target table
[I 221110 19:29:01 insert_yamashita_stars:139] obj_id ra ... updated_at target_id
10 4881345510044485120 73.042113 ... 2022-04-30 02:07:08.500763 77187

[1 rows x 58 columns]
[I 221110 19:29:01 insert_yamashita_stars:140] <class 'pandas.core.frame.DataFrame'>
[I 221110 19:29:01 insert_yamashita_stars:150] Close the database.
obj_id ra ... priority effective_exptime
0 2536159496590552704 15.842504 ... 1.0 450.0
1 5176546064064586624 37.071596 ... 1.0 450.0
2 139724391470489472 42.229021 ... 1.0 450.0
3 892231562565363072 111.969802 ... 1.0 450.0
4 3097940536010212736 123.786591 ... 1.0 450.0
5 3830980604624181376 156.128799 ... 1.0 450.0
6 6910475935427725824 315.461113 ... 1.0 450.0
7 2644572064644349952 352.422169 ... 1.0 450.0
8 2881271732415859072 357.934477 ... 1.0 450.0
9 5057499161282786432 53.137007 ... 1.0 450.0
10 4881345510044485120 73.042113 ... 1.0 10.0

[11 rows x 31 columns]
obj_id ra dec ... filter_z filter_y filter_j
0 2536159496590552704 15.842504 -0.346611 ... z_ps1 y_ps1 None
1 5176546064064586624 37.071596 -8.454528 ... z_ps1 y_ps1 None
2 139724391470489472 42.229021 33.763401 ... z_ps1 y_ps1 None
3 892231562565363072 111.969802 32.237791 ... z_ps1 y_ps1 None
4 3097940536010212736 123.786591 7.529382 ... z_ps1 None None
5 3830980604624181376 156.128799 -0.535322 ... z_ps1 y_ps1 None
6 6910475935427725824 315.461113 -5.764211 ... z_ps1 y_ps1 None
7 2644572064644349952 352.422169 0.185435 ... z_ps1 y_ps1 None
8 2881271732415859072 357.934477 37.928491 ... z_ps1 y_ps1 None
9 5057499161282786432 53.137007 -27.863433 ... None None None
10 4881345510044485120 73.042113 -27.064268 ... z_sdss None None

[11 rows x 39 columns]
Loading