feat: add a script to insert a new proposal id for 2022sep commissioning

examples/commissioning_2022sep/database_work/insert_yamashita_stars.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+
+import argparse
+import datetime
+
+import numpy as np
+import pandas as pd
+import toml
+from astropy.table import Table
+from logzero import logger
+from targetdb import targetdb
+
+from targetdb_api_utils import insert_simple
+from targetdb_api_utils import join_backref_values
+
+
+def load_data(infile):
+    tb = Table.read(infile, format="ascii.ecsv")
+    proposal_id = tb.meta["proposal_id"]
+    df = tb.to_pandas()
+    print(df)
+    return df, proposal_id
+
+
+def format_data(df, proposal_id):
+
+    filter_bands = ["g", "r", "i", "z", "y", "j"]
+
+    dfout = df.copy(deep=True)
+
+    dfout.rename(
+        columns={"input_catalog": "input_catalog_name"},
+        inplace=True,
+    )
+    dfout["proposal_id"] = proposal_id
+    dfout["target_type_name"] = "SCIENCE"
+
+    lists_filtername = {"g": [], "r": [], "i": [], "z": [], "y": [], "j": []}
+
+    for i in range(df.index.size):
+
+        filter_name = df["filter_name"][i]
+
+        if "APASS" in filter_name:
+            filter_system = "sdss"
+        elif "PS1" in filter_name:
+            filter_system = "ps1"
+        else:
+            logger.error(f"Filter system is not registered. Check the code.")
+            exit()
+
+        for band in filter_bands:
+            try:
+                if np.isfinite(dfout[f"psf_mag_{band}"][i]) and np.isfinite(
+                    dfout[f"psf_flux_{band}"][i]
+                ):
+                    lists_filtername[band].append(f"{band}_{filter_system}")
+                else:
+                    logger.warn(
+                        f"NaN is found in the {band} flux and magnitude for the object {df['obj_id'][i]}. None is put there."
+                    )
+                    lists_filtername[band].append(None)
+            except KeyError as e:
+                logger.warn(
+                    f"No {band} information is found in the data and the columns are set as NULL in the database."
+                )
+                lists_filtername[band].append(None)
+
+    for band in filter_bands:
+        dfout[f"filter_{band}"] = lists_filtername[band]
+
+    print(dfout)
+
+    return dfout
+
+
+def main(conf, infile, dry_run=False):
+
+    logger.info(f"Loading input data into dataframe")
+    df, proposal_id = load_data(infile)
+
+    df = format_data(df, proposal_id)
+
+    logger.info(f"Load config file {conf}.")
+    config = toml.load(conf)
+
+    logger.info(f"Connect to the database.")
+    db = targetdb.TargetDB(**dict(config["targetdb"]))
+    db.connect()
+
+    backref_tables = ["proposal", "input_catalog", "target_type"]
+    backref_keys = ["proposal_id", "input_catalog_name", "target_type_name"]
+    backref_check_keys = ["proposal_id", "input_catalog_id", "target_type_id"]
+
+    for i in range(len(backref_tables)):
+        df = join_backref_values(
+            df,
+            db=db,
+            table=backref_tables[i],
+            key=backref_keys[i],
+            check_key=backref_check_keys[i],
+        )
+
+    logger.debug(df[["proposal_id", "input_catalog_name", "target_type_name"]])
+
+    if dry_run:
+        logger.info("Dry run. Do nothing.")
+        pass
+    else:
+        logger.info(f"Insert data into the database.")
+        db = insert_simple(db, table="target", df=df, fetch_table=False)
+
+    logger.info(f"Close the database.")
+    db.close()
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "conf",
+        type=str,
+        help="Config file for the script to run. Must be a .toml file (mandatory)",
+    )
+
+    parser.add_argument(
+        "--infile",
+        type=str,
+        default="../../../external_data/commissioning_2022sep/stars_yamashita/targets_S22B-EN16.ecsv",
+        help="Input file from Yamashita-san",
+    )
+
+    parser.add_argument(
+        "--dry_run",
+        action="store_true",
+        help="Dry-run (no update of the database)",
+    )
+
+    args = parser.parse_args()
+
+    main(args.conf, args.infile, args.dry_run)

examples/commissioning_2022sep/database_work/populate_targetdb.sh

@@ -0,0 +1,134 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+PY_SCRIPT="targetdb_api_utils.py"
+
+echo "Working on ${HOSTNAME}"
+
+if [ $HOSTNAME == "pfsa-usr01-gb.subaru.nao.ac.jp" ]; then
+    DB_CONF_FILE="../../../database_configs/config_pfsa-db01-gb_commissioning_2022may.toml"
+else
+    DB_CONF_FILE="../../../database_configs/config_local.toml"
+fi
+
+# STARDATA_DIR="../../../external_data/commissioning_2022may/fluxstd_ishigaki/feather/"
+# STARDATA_PREFIX="fluxstd_v1.0"
+
+# SKYDATA_DIR="../../../external_data/commissioning_2022may/sky_murata/feather/"
+# SKYDATA_PREFIX=""
+
+# COSMOLOGY_DIR="../../../external_data/commissioning_2022jun/cosmology/"
+
+MISCDATA_DIR="../../../external_data/commissioning_2022may/misc/"
+
+# populate tables other than fluxstd and target by resetting all tables
+python3 ./${PY_SCRIPT} ${DB_CONF_FILE} \
+    --skip_proposal_category \
+    --proposal ${MISCDATA_DIR}/proposal.csv \
+    --skip_input_catalog \
+    --skip_target_type \
+    --skip_target \
+    --skip_fluxstd \
+    --skip_sky
+
+# RESET_ALL=false
+
+# # populate the target table
+# for target in ${COSMOLOGY_DIR}/*.fits; do
+#     if [ -f $target ]; then
+#         python ./${PY_SCRIPT} ${DB_CONF_FILE} \
+#             --skip_proposal_category \
+#             --skip_proposal \
+#             --skip_input_catalog \
+#             --skip_target_type \
+#             --skip_fluxstd \
+#             --skip_sky \
+#             --target $target #--reset target \
+#     fi
+# done
+
+# while true; do
+#     read -p "Do you wish to reset ALL tables in targetDB? [y(es)/n(o)]" yn
+#     case $yn in
+#     [Yy]*)
+#         RESET_ALL=true
+#         break
+#         ;;
+#     [Nn]*)
+#         RESET_ALL=false
+#         break
+#         ;;
+#     *) echo "Please answer yes or no." ;;
+#     esac
+# done
+
+# if [ ${RESET_ALL} = true ]; then
+#     echo "RESETTING ALL TABLES"
+#     python ./${PY_SCRIPT} ${DB_CONF_FILE} \
+#         --reset all \
+#         --skip_proposal_category \
+#         --skip_proposal \
+#         --skip_input_catalog \
+#         --skip_target_type \
+#         --skip_target \
+#         --skip_fluxstd \
+#         --skip_sky
+# fi
+
+# while true; do
+#     read -p "Do you wish to ingest data into proposal_category, proposal, input_catalog, and target_type tables in targetDB? [y(es)/n(o)]" yn
+#     case $yn in
+#     [Yy]*)
+#         INSERT_MISC=true
+#         break
+#         ;;
+#     [Nn]*)
+#         INSERT_MISC=false
+#         break
+#         ;;
+#     *) echo "Please answer yes or no." ;;
+#     esac
+# done
+
+# # populate tables other than fluxstd and target by resetting all tables
+# if [ $INSERT_MISC == true ]; then
+#     python ./${PY_SCRIPT} ${DB_CONF_FILE} \
+#         --proposal_category ${MISCDATA_DIR}/proposal_category.csv \
+#         --proposal ${MISCDATA_DIR}/proposal.csv \
+#         --input_catalog ${MISCDATA_DIR}/input_catalog.csv \
+#         --target_type ${MISCDATA_DIR}/target_type.csv \
+#         --skip_target \
+#         --skip_fluxstd \
+#         --skip_sky
+# fi
+
+# # populate the sky table
+# for sky in ${SKYDATA_DIR}/*.feather; do
+#     if [ -f $sky ]; then
+#         echo $sky
+#         python ./${PY_SCRIPT} ${DB_CONF_FILE} \
+#             --reset sky \
+#             --skip_proposal_category \
+#             --skip_proposal \
+#             --skip_input_catalog \
+#             --skip_target_type \
+#             --skip_target \
+#             --skip_fluxstd \
+#             --sky $sky
+#     fi
+# done
+
+# # populate the fluxstd table
+# for fluxstd in ${STARDATA_DIR}/${STARDATA_PREFIX}*.feather; do
+#     if [ -f $fluxstd ]; then
+#         python ./${PY_SCRIPT} ${DB_CONF_FILE} \
+#             --skip_proposal_category \
+#             --skip_proposal \
+#             --skip_input_catalog \
+#             --skip_target_type \
+#             --skip_target \
+#             --skip_sky \
+#             --fluxstd $fluxstd
+#     fi
+# done