From 0be6981bca8b293bf0c289469b5a02b78960d3ea Mon Sep 17 00:00:00 2001 From: Bill-Gray Date: Sun, 11 Feb 2024 13:12:09 -0500 Subject: [PATCH] 'csv2ades.c' (code to convert Gaia asteroid astrometry from .csv to ADES XML) heavily revised. It now figures out which file(s) are needed, provides more intelligent error messages, and the documentation at the top of the file is less confusing. The resulting ADES files will pass the XML validation. --- csv2ades.c | 136 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 87 insertions(+), 49 deletions(-) diff --git a/csv2ades.c b/csv2ades.c index d7dac74..0411e2f 100644 --- a/csv2ades.c +++ b/csv2ades.c @@ -1,55 +1,58 @@ -/* Code to process Gaia-DR2 observations from CSV files at +/* Code to process Gaia asteroid observation data into ADES format. -http://cdn.gea.esac.esa.int/Gaia/gdr2/sso_observation/csv/ - -_or_ Gaia-DR3 observations from somewhat differently formatted files at + Gaia has provided asteroid astrometry in three releases, each time +in slightly different CSV formats that are non-standard and a little +oddly organized. The most recent (and generally preferred) data are +in the FPR (Focussed Product Release) : -http://cdn.gea.esac.esa.int/Gaia/gdr3/Solar_system/sso_observation/ +https://cdn.gea.esac.esa.int/Gaia/gfpr/Solar_system/sso_observation/ -_or_ Gaia-FPR (Focussed Product Release) files at + These do lack photometry, but the preceding Gaia-DR3 files, -https://cdn.gea.esac.esa.int/Gaia/gfpr/Solar_system/sso_observation/ +http://cdn.gea.esac.esa.int/Gaia/gdr3/Solar_system/sso_observation/ - into the ADES format. Run as, for example, + provide that. This program no longer processes the eldest Gaia-DR2 +data from -./csv2ades input_file.csv > output.ades -./csv2ades -4179,101955 input_file.csv > output.ades +http://cdn.gea.esac.esa.int/Gaia/gdr2/sso_observation/csv/ - The first example would output all the observations from -'input_file.csv'. The second example would output only observations -for objects (4179) Toutatis to (101955) Bennu. + To use this program, download and un-GZip the necessary CSV file(s) +from the above sites. Run this program as, for example, - Further notes : there are four GZIPped files in the above directory -for DR2, ranging from about 50 to 78 MBytes compressed and expanding -to 150 to 230 MBytes uncompressed. Each contains astrometry for a -specific range of numbered minor planets; for example, the first has -data for objects (8) Flora to (4435) Holt. +./csv2ades 4179 > output.ades +./csv2ades 4179,101955 > output.ades -SsoObservation_-4284967216_-4284922946.csv.gz 8 - 4435 -SsoObservation_-4284922936_-4284857966.csv.gz 4436 - 10933 -SsoObservation_-4284857946_-4284702156.csv.gz 10935 - 26514 -SsoObservation_-4284702096_-4283326086.csv.gz 26520 - 164121 + The first example would output all the observations for (4179) +Toutatis to 'output.ades'. The second example would output observations +for all objects from (4179) Toutatis to (101955) Bennu, inclusive. +Note that the output files can get to be pretty big. You may be +better off just running an object or a few at a time. - The Gaia DR3 and FPR data are less obviously arranged and are larger. -In what I assume is an effort to make the files of similar size, -numbered object (N) is placed in the file + There are 20 GZIPped files in the above directory for FPR, each about +1.3 GBytes, 27 GBytes for all twenty. Each file contains astrometry for +about 1/20 of the numbered minor planets observed by Gaia. Numbered +object (N) is placed in the file SsoObservation_xx.csv where xx = (N + 11) mod 20. Thus, for example, the data for (118186) would be in the file SsoObservation_17.csv. The reason -for the offset by 11 eludes me. */ +for the offset by 11 eludes me. It does mean that if you're looking +for a specific object, you can figure out which file will have data +for it and then download only that file. (Or you can run the above +command and get an error message telling you which file wasn't found.) */ #include #include #include #include #include +#include #include "watdefs.h" #include "date.h" static char *get_csv( char *obuff, const char *ibuff, - const char *header, const char *tag) + const char *header, const char *tag, const size_t max_len) { size_t i; unsigned nval = 0; @@ -69,7 +72,7 @@ static char *get_csv( char *obuff, const char *ibuff, ibuff++; } assert( ibuff); - for( i = 0; ibuff[i] && ibuff[i] != ','; i++) + for( i = 0; i < max_len && ibuff[i] && ibuff[i] != ','; i++) obuff[i] = ibuff[i]; obuff[i] = '\0'; return( obuff); @@ -81,7 +84,7 @@ static void csv_to_ades( const char *ibuff, const char *header, char tbuff[90]; int num; - get_csv( tbuff, ibuff, header, "number_mp,"); + get_csv( tbuff, ibuff, header, "number_mp,", 9); num = atoi( tbuff); if( num >= low_num && num <= high_num) { @@ -94,31 +97,31 @@ static void csv_to_ades( const char *ibuff, const char *header, printf( " 258\n"); printf( " ICRF_AU\n"); printf( " 0\n"); - printf( " %s\n", get_csv( tbuff, ibuff, header, "x_gaia,")); - printf( " %s\n", get_csv( tbuff, ibuff, header, "y_gaia,")); - printf( " %s\n", get_csv( tbuff, ibuff, header, "z_gaia,")); - jd_obs = atof( get_csv( tbuff, ibuff, header, "epoch_utc,")) + jd_epoch; + printf( " %s\n", get_csv( tbuff, ibuff, header, "x_gaia,", 13)); + printf( " %s\n", get_csv( tbuff, ibuff, header, "y_gaia,", 13)); + printf( " %s\n", get_csv( tbuff, ibuff, header, "z_gaia,", 13)); + jd_obs = atof( get_csv( tbuff, ibuff, header, "epoch_utc,", 99)) + jd_epoch; full_ctime( tbuff, jd_obs, FULL_CTIME_MILLISECS | FULL_CTIME_YMD | FULL_CTIME_LEADING_ZEROES | FULL_CTIME_MONTHS_AS_DIGITS); tbuff[4] = tbuff[7] = '-'; tbuff[10] = 'T'; printf( " %sZ\n", tbuff); - printf( " %s\n", get_csv( tbuff, ibuff, header, "ra,")); - printf( " %s\n", get_csv( tbuff, ibuff, header, "dec,")); - printf( " %f\n", atof( get_csv( tbuff, ibuff, header, "ra_error_random,")) / 1000.); - printf( " %f\n", atof( get_csv( tbuff, ibuff, header, "dec_error_random,")) / 1000.); - printf( " %s\n", get_csv( tbuff, ibuff, header, "ra_dec_correlation_random,")); + printf( " %.9f\n", atof( get_csv( tbuff, ibuff, header, "ra,", 99))); + printf( " %.9f\n", atof( get_csv( tbuff, ibuff, header, "dec,", 99))); + printf( " %.5f\n", atof( get_csv( tbuff, ibuff, header, "ra_error_random,", 99)) / 1000.); + printf( " %.5f\n", atof( get_csv( tbuff, ibuff, header, "dec_error_random,", 99)) / 1000.); + printf( " %s\n", get_csv( tbuff, ibuff, header, "ra_dec_correlation_random,", 10)); printf( " Gaia3\n"); if( strstr( header, "g_mag,")) /* FPR doesn't have magnitude data */ { - get_csv( tbuff, ibuff, header, "g_mag,"); + get_csv( tbuff, ibuff, header, "g_mag,", 9); if( *tbuff >= '0' && *tbuff <= '9') { double g_flux, g_flux_sigma; printf( " %s\n", tbuff); - g_flux = atof( get_csv( tbuff, ibuff, header, "g_flux")); - g_flux_sigma = atof( get_csv( tbuff, ibuff, header, "g_flux_error")); + g_flux = atof( get_csv( tbuff, ibuff, header, "g_flux", 99)); + g_flux_sigma = atof( get_csv( tbuff, ibuff, header, "g_flux_error", 99)); printf( " %.3f\n", (g_flux_sigma / g_flux) / log( 2.512)); printf( " G\n"); @@ -129,30 +132,65 @@ static void csv_to_ades( const char *ibuff, const char *header, } } +static void error_exit( void) +{ + fprintf( stderr, "Run as 'csv2ades (asteroid number)' for ADES data on a single\n" + "object, or 'csv2ades (lownum,highnum)' for ADES data for a range\n" + "of numbered objects. Output is to stdout. See 'csv2ades.c' for\n" + "information about where to get the Gaia data in CSV files, needed\n" + "as input to this program.\n"); + exit( -1); +} + +/* See above comments about the layout of numbered observations in the +various SsoObservation_XX.csv files. Unless you ask for more than 20 +asteroids, we don't need to look at all twenty files. */ + +static bool file_needed( const int file_no, const int low_num, const int high_num) +{ + int i; + + for( i = low_num; i <= high_num && i < low_num + 20; i++) + if( ((i + 11) % 20) == file_no) + return true; + return( false); +} + int main( const int argc, const char **argv) { FILE *hdr_file = fopen( "gaia.hdr", "rb"); char buff[800]; - int i, low_num = 1, high_num = 1000000; + int low_num, high_num = -1, file_num; if( !hdr_file) { perror( "Can't open 'gaia.hdr'"); return( -1); } - for( i = 1; i < argc; i++) - if( argv[i][0] == '-') - sscanf( argv[i] + 1, "%d,%d", &low_num, &high_num); + if( argc != 2 || sscanf( argv[1], "%d,%d", &low_num, &high_num) < 1) + { + fprintf( stderr, "No asteroids specified on the command line\n"); + error_exit( ); + } + if( high_num < 0) /* just specified one asteroid */ + high_num = low_num; while( fgets( buff, sizeof( buff), hdr_file)) if( *buff != '*') printf( "%s", buff); - else for( i = 1; i < argc; i++) - if( argv[i][0] != '-') + else for( file_num = 0; file_num < 20; file_num++) + if( file_needed( file_num, low_num, high_num)) { - FILE *ifile = fopen( argv[i], "rb"); + FILE *ifile; char header[800]; - assert( ifile); + snprintf( buff, sizeof( buff), "SsoObservation_%02d.csv", file_num); + ifile = fopen( buff, "rb"); + if( !ifile) + { + fprintf( stderr, "File '%s' not found\n" + "'csv2ades.c' contains directions on where to get Gaia data\n", buff); + exit( -2); + } *header = '\0'; while( fgets( buff, sizeof( buff), ifile)) {