diff -Nru starjava-ttools-3.4.5/build.xml starjava-ttools-3.4.6/build.xml --- starjava-ttools-3.4.5/build.xml 2022-06-10 13:14:05.000000000 +0000 +++ starjava-ttools-3.4.6/build.xml 2022-07-08 09:59:19.000000000 +0000 @@ -80,7 +80,7 @@ - + diff -Nru starjava-ttools-3.4.5/debian/changelog starjava-ttools-3.4.6/debian/changelog --- starjava-ttools-3.4.5/debian/changelog 2022-06-13 14:59:22.000000000 +0000 +++ starjava-ttools-3.4.6/debian/changelog 2022-07-16 08:41:13.000000000 +0000 @@ -1,3 +1,20 @@ +starjava-ttools (3.4.6-2) unstable; urgency=medium + + * Update starjava-tjoin and starjava-util binary dependency minversions + + -- Ole Streicher Sat, 16 Jul 2022 10:41:13 +0200 + +starjava-ttools (3.4.6-1) unstable; urgency=medium + + * New upstream version 3.4.6 + * Rediff patches + * Update minversion for starjava-tjoin and starjava-util + * Push Standards-Version to 4.6.1. No changes needed + * remove tjoins.NdRange references + * tjoin: add MatchEngine.createCoverageFactory + + -- Ole Streicher Thu, 14 Jul 2022 19:35:46 +0200 + starjava-ttools (3.4.5-1) unstable; urgency=medium * New upstream version 3.4.5 diff -Nru starjava-ttools-3.4.5/debian/control starjava-ttools-3.4.6/debian/control --- starjava-ttools-3.4.5/debian/control 2022-06-13 14:59:22.000000000 +0000 +++ starjava-ttools-3.4.6/debian/control 2022-07-16 08:40:12.000000000 +0000 @@ -35,11 +35,11 @@ starlink-registry-java, starlink-table-java (>= 4.1~), starlink-task-java, - starlink-tjoin-java (>=1.0.+2022.05.13~), - starlink-util-java (>= 1.0+2022.04.04~), + starlink-tjoin-java (>=1.0.+2022.07.13~), + starlink-util-java (>= 1.0+2022.06.20~), starlink-vo-java (>= 0.2+2021.11.02~), starlink-votable-java (>= 2.0+2022.04.04~) -Standards-Version: 4.6.0 +Standards-Version: 4.6.1 Vcs-Browser: https://salsa.debian.org/debian-astro-team/starjava-ttools Vcs-Git: https://salsa.debian.org/debian-astro-team/starjava-ttools.git Homepage: http://www.starlink.ac.uk/stilts/ @@ -107,8 +107,8 @@ Package: starlink-ttools-java Architecture: all Depends: starlink-table-java (>= 4.1~), - starlink-tjoin-java (>=1.0.+2022.04.14~), - starlink-util-java (>= 1.0+2022.04.04~), + starlink-tjoin-java (>=1.0.+2022.07.13~), + starlink-util-java (>= 1.0+2022.06.20~), ${java:Depends}, ${misc:Depends} Breaks: starlink-topcat-java (<< 4.8.4~) diff -Nru starjava-ttools-3.4.5/debian/patches/Clearly-mark-this-version-as-Debian.patch starjava-ttools-3.4.6/debian/patches/Clearly-mark-this-version-as-Debian.patch --- starjava-ttools-3.4.5/debian/patches/Clearly-mark-this-version-as-Debian.patch 2022-06-13 14:59:22.000000000 +0000 +++ starjava-ttools-3.4.6/debian/patches/Clearly-mark-this-version-as-Debian.patch 2022-07-14 17:35:39.000000000 +0000 @@ -8,7 +8,7 @@ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/build.xml b/build.xml -index ef443d2..f63c728 100644 +index 97b0564..1ce7936 100644 --- a/build.xml +++ b/build.xml @@ -487,7 +487,7 @@ @@ -39,10 +39,10 @@ diff --git a/src/docs/sun256.xml b/src/docs/sun256.xml -index fb4c22b..df23944 100644 +index 8ad39d2..6c1d03c 100644 --- a/src/docs/sun256.xml +++ b/src/docs/sun256.xml -@@ -230,6 +230,10 @@ detailed on-line help is available from the tools themselves. +@@ -231,6 +231,10 @@ detailed on-line help is available from the tools themselves. under the GNU Lesser General Public License (LGPL). @@ -53,7 +53,7 @@ -@@ -367,6 +371,34 @@ here are a few possibilities: +@@ -369,6 +373,34 @@ here are a few possibilities:

diff -Nru starjava-ttools-3.4.5/debian/patches/Create-manpages.patch starjava-ttools-3.4.6/debian/patches/Create-manpages.patch --- starjava-ttools-3.4.5/debian/patches/Create-manpages.patch 2022-06-13 14:59:22.000000000 +0000 +++ starjava-ttools-3.4.6/debian/patches/Create-manpages.patch 2022-07-14 17:35:39.000000000 +0000 @@ -9,7 +9,7 @@ create mode 100644 src/main/uk/ac/starlink/ttools/build/UsageWriterMan.java diff --git a/build.xml b/build.xml -index 944be7e..5fe490a 100644 +index 153dbae..eb1f4b4 100644 --- a/build.xml +++ b/build.xml @@ -1393,6 +1393,13 @@ diff -Nru starjava-ttools-3.4.5/debian/patches/Don-t-build-StiltsServer.patch starjava-ttools-3.4.6/debian/patches/Don-t-build-StiltsServer.patch --- starjava-ttools-3.4.5/debian/patches/Don-t-build-StiltsServer.patch 2022-06-13 14:59:22.000000000 +0000 +++ starjava-ttools-3.4.6/debian/patches/Don-t-build-StiltsServer.patch 2022-07-14 17:35:39.000000000 +0000 @@ -12,7 +12,7 @@ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/build.xml b/build.xml -index 7cdb1b5..412939d 100644 +index 5ffe381..0d41cce 100644 --- a/build.xml +++ b/build.xml @@ -438,6 +438,8 @@ @@ -34,10 +34,10 @@ diff --git a/src/docs/sun256.xml b/src/docs/sun256.xml -index f78f49f..fb4c22b 100644 +index 652f90a..8ad39d2 100644 --- a/src/docs/sun256.xml +++ b/src/docs/sun256.xml -@@ -93,7 +93,6 @@ +@@ -94,7 +94,6 @@ @@ -45,7 +45,7 @@ -@@ -320,7 +319,6 @@ same ground as the STIL and TOPCAT user documents +@@ -322,7 +321,6 @@ same ground as the STIL and TOPCAT user documents

calc, funcs, @@ -53,7 +53,7 @@ xsdvalidate.

-@@ -7116,7 +7114,9 @@ If you have requirements which are not currently provided, please +@@ -7264,7 +7262,9 @@ If you have requirements which are not currently provided, please contact the author for discussion.

@@ -65,10 +65,10 @@ Examples diff --git a/src/main/uk/ac/starlink/ttools/Stilts.java b/src/main/uk/ac/starlink/ttools/Stilts.java -index f211f7e..fd197fa 100644 +index e50ac73..9aa81a0 100644 --- a/src/main/uk/ac/starlink/ttools/Stilts.java +++ b/src/main/uk/ac/starlink/ttools/Stilts.java -@@ -158,7 +158,6 @@ public class Stilts { +@@ -159,7 +159,6 @@ public class Stilts { taskFactory_.register( "plot3d", taskPkg + "TablePlot3D" ); taskFactory_.register( "plothist", taskPkg + "TableHistogram" ); taskFactory_.register( "regquery", taskPkg + "RegQuery" ); diff -Nru starjava-ttools-3.4.5/debian/patches/Don-t-include-in-out-documents-that-are-not-packaged.patch starjava-ttools-3.4.6/debian/patches/Don-t-include-in-out-documents-that-are-not-packaged.patch --- starjava-ttools-3.4.5/debian/patches/Don-t-include-in-out-documents-that-are-not-packaged.patch 2022-06-13 14:59:22.000000000 +0000 +++ starjava-ttools-3.4.6/debian/patches/Don-t-include-in-out-documents-that-are-not-packaged.patch 2022-07-14 17:35:39.000000000 +0000 @@ -7,7 +7,7 @@ 1 file changed, 10 deletions(-) diff --git a/src/docs/sun256.xml b/src/docs/sun256.xml -index 7391d5a..0faefd9 100644 +index d5196d0..315551e 100644 --- a/src/docs/sun256.xml +++ b/src/docs/sun256.xml @@ -46,15 +46,9 @@ diff -Nru starjava-ttools-3.4.5/debian/patches/Fix-build.xml-for-use-outside-of-starjava.patch starjava-ttools-3.4.6/debian/patches/Fix-build.xml-for-use-outside-of-starjava.patch --- starjava-ttools-3.4.5/debian/patches/Fix-build.xml-for-use-outside-of-starjava.patch 2022-06-13 14:59:22.000000000 +0000 +++ starjava-ttools-3.4.6/debian/patches/Fix-build.xml-for-use-outside-of-starjava.patch 2022-07-14 17:35:39.000000000 +0000 @@ -25,7 +25,7 @@ 1 file changed, 87 insertions(+), 53 deletions(-) diff --git a/build.xml b/build.xml -index c19ca92..ff636b8 100644 +index 69cecc2..64deb83 100644 --- a/build.xml +++ b/build.xml @@ -33,12 +33,6 @@ @@ -63,10 +63,10 @@ - - -- +- + + -+ ++ diff -Nru starjava-ttools-3.4.5/debian/patches/Generate-bash-autocompletion-script.patch starjava-ttools-3.4.6/debian/patches/Generate-bash-autocompletion-script.patch --- starjava-ttools-3.4.5/debian/patches/Generate-bash-autocompletion-script.patch 2022-06-13 14:59:22.000000000 +0000 +++ starjava-ttools-3.4.6/debian/patches/Generate-bash-autocompletion-script.patch 2022-07-14 17:35:39.000000000 +0000 @@ -10,7 +10,7 @@ create mode 100644 src/main/uk/ac/starlink/ttools/build/BashComplete.java diff --git a/build.xml b/build.xml -index 5fe490a..ef443d2 100644 +index eb1f4b4..97b0564 100644 --- a/build.xml +++ b/build.xml @@ -1400,6 +1400,14 @@ diff -Nru starjava-ttools-3.4.5/debian/patches/Package-jystilts.patch starjava-ttools-3.4.6/debian/patches/Package-jystilts.patch --- starjava-ttools-3.4.5/debian/patches/Package-jystilts.patch 2022-06-13 14:59:22.000000000 +0000 +++ starjava-ttools-3.4.6/debian/patches/Package-jystilts.patch 2022-07-14 17:35:39.000000000 +0000 @@ -8,10 +8,10 @@ 2 files changed, 38 insertions(+), 51 deletions(-) diff --git a/src/docs/sun256.xml b/src/docs/sun256.xml -index df23944..7391d5a 100644 +index 6c1d03c..d5196d0 100644 --- a/src/docs/sun256.xml +++ b/src/docs/sun256.xml -@@ -1494,53 +1494,15 @@ is given in the following subsections. +@@ -1497,53 +1497,15 @@ is given in the following subsections. Running JyStilts diff -Nru starjava-ttools-3.4.5/debian/patches/Remove-build-of-UCD-and-VOUnit-validation-harnesses.patch starjava-ttools-3.4.6/debian/patches/Remove-build-of-UCD-and-VOUnit-validation-harnesses.patch --- starjava-ttools-3.4.5/debian/patches/Remove-build-of-UCD-and-VOUnit-validation-harnesses.patch 2022-06-13 14:59:22.000000000 +0000 +++ starjava-ttools-3.4.6/debian/patches/Remove-build-of-UCD-and-VOUnit-validation-harnesses.patch 2022-07-14 17:35:39.000000000 +0000 @@ -13,7 +13,7 @@ 6 files changed, 21 insertions(+), 38 deletions(-) diff --git a/build.xml b/build.xml -index f63c728..79b588d 100644 +index 1ce7936..19a226f 100644 --- a/build.xml +++ b/build.xml @@ -477,6 +477,13 @@ @@ -197,10 +197,10 @@ return map; } diff --git a/src/testcases/uk/ac/starlink/ttools/func/FuncTest.java b/src/testcases/uk/ac/starlink/ttools/func/FuncTest.java -index b257f67..9e224c1 100644 +index 44c2488..5784c17 100644 --- a/src/testcases/uk/ac/starlink/ttools/func/FuncTest.java +++ b/src/testcases/uk/ac/starlink/ttools/func/FuncTest.java -@@ -945,23 +945,6 @@ public class FuncTest extends TestCase { +@@ -962,23 +962,6 @@ public class FuncTest extends TestCase { checkHealpix( 9, 1110787 ); } diff -Nru starjava-ttools-3.4.5/debian/patches/Remove-Plastic-references.patch starjava-ttools-3.4.6/debian/patches/Remove-Plastic-references.patch --- starjava-ttools-3.4.5/debian/patches/Remove-Plastic-references.patch 2022-06-13 14:59:22.000000000 +0000 +++ starjava-ttools-3.4.6/debian/patches/Remove-Plastic-references.patch 2022-07-14 17:35:39.000000000 +0000 @@ -9,7 +9,7 @@ 3 files changed, 3 insertions(+), 32 deletions(-) diff --git a/build.xml b/build.xml -index 412939d..c19ca92 100644 +index 0d41cce..69cecc2 100644 --- a/build.xml +++ b/build.xml @@ -440,6 +440,7 @@ @@ -29,10 +29,10 @@ diff --git a/src/main/uk/ac/starlink/ttools/Stilts.java b/src/main/uk/ac/starlink/ttools/Stilts.java -index fd197fa..317d157 100644 +index 9aa81a0..c8c7e8b 100644 --- a/src/main/uk/ac/starlink/ttools/Stilts.java +++ b/src/main/uk/ac/starlink/ttools/Stilts.java -@@ -215,7 +215,6 @@ public class Stilts { +@@ -216,7 +216,6 @@ public class Stilts { modeFactory_.register( "discard", modePkg + "NullMode" ); modeFactory_.register( "topcat", modePkg + "TopcatMode" ); modeFactory_.register( "samp", modePkg + "SampMode" ); diff -Nru starjava-ttools-3.4.5/debian/patches/Remove-reference-to-HTM-pixellation.patch starjava-ttools-3.4.6/debian/patches/Remove-reference-to-HTM-pixellation.patch --- starjava-ttools-3.4.5/debian/patches/Remove-reference-to-HTM-pixellation.patch 2022-06-13 14:59:22.000000000 +0000 +++ starjava-ttools-3.4.6/debian/patches/Remove-reference-to-HTM-pixellation.patch 2022-07-14 17:35:39.000000000 +0000 @@ -13,7 +13,7 @@ 6 files changed, 4 insertions(+), 48 deletions(-) diff --git a/build.xml b/build.xml -index 42ac6b8..7cdb1b5 100644 +index 2953961..5ffe381 100644 --- a/build.xml +++ b/build.xml @@ -437,6 +437,7 @@ diff -Nru starjava-ttools-3.4.5/debian/patches/series starjava-ttools-3.4.6/debian/patches/series --- starjava-ttools-3.4.5/debian/patches/series 2022-01-05 16:27:51.000000000 +0000 +++ starjava-ttools-3.4.6/debian/patches/series 2022-07-14 17:35:39.000000000 +0000 @@ -16,3 +16,5 @@ Fix-import-path-for-jfree.svg.patch Don-t-include-in-out-documents-that-are-not-packaged.patch Remove-build-of-UCD-and-VOUnit-validation-harnesses.patch +tjoin-remove-NdRange-and-all-references-to-it.patch +tjoin-add-MatchEngine.createCoverageFactory.patch diff -Nru starjava-ttools-3.4.5/debian/patches/tjoin-add-MatchEngine.createCoverageFactory.patch starjava-ttools-3.4.6/debian/patches/tjoin-add-MatchEngine.createCoverageFactory.patch --- starjava-ttools-3.4.5/debian/patches/tjoin-add-MatchEngine.createCoverageFactory.patch 1970-01-01 00:00:00.000000000 +0000 +++ starjava-ttools-3.4.6/debian/patches/tjoin-add-MatchEngine.createCoverageFactory.patch 2022-07-14 17:35:39.000000000 +0000 @@ -0,0 +1,71 @@ +From: Mark Taylor +Date: Wed, 8 Jun 2022 16:51:09 +0100 +Subject: tjoin: add MatchEngine.createCoverageFactory + +This method is intended as a replacement for the use of NdRange. +Implementations are in place, but at this commit it is not used. + +This is part of https://github.com/Starlink/starjava/commit/51b5c938 +--- + .../ac/starlink/ttools/join/HumanMatchEngine.java | 31 +++++++++++++++++++++- + 1 file changed, 30 insertions(+), 1 deletion(-) + +diff --git a/src/testcases/uk/ac/starlink/ttools/join/HumanMatchEngine.java b/src/testcases/uk/ac/starlink/ttools/join/HumanMatchEngine.java +index 07aa1af..8df28f6 100644 +--- a/src/testcases/uk/ac/starlink/ttools/join/HumanMatchEngine.java ++++ b/src/testcases/uk/ac/starlink/ttools/join/HumanMatchEngine.java +@@ -1,17 +1,19 @@ + package uk.ac.starlink.ttools.join; + ++import java.util.function.Predicate; + import java.util.function.Supplier; + import java.util.logging.Logger; + import uk.ac.starlink.table.DefaultValueInfo; + import uk.ac.starlink.table.DescribedValue; + import uk.ac.starlink.table.ValueInfo; ++import uk.ac.starlink.table.join.Coverage; + import uk.ac.starlink.table.join.MatchEngine; + import uk.ac.starlink.table.join.MatchKit; + import uk.ac.starlink.ttools.func.CoordsRadians; + + /** + * MatchEngine adaptor which transforms the base engine so that it +- * uses more human-friendly units. Currently, this means that it uses ++ * uses more human-friendly units. Currently, this means that it + * eschews radians in favour of degrees or arcseconds for angular quantities; + * it decides which on the basis of UCDs. + * In other respects, this engine will behave +@@ -132,6 +134,33 @@ public class HumanMatchEngine implements MatchEngine { + }; + } + ++ public Supplier createCoverageFactory() { ++ Supplier baseFact = baseEngine_.createCoverageFactory(); ++ return () -> new Coverage() { ++ final Coverage baseCoverage = baseFact.get(); ++ public boolean isEmpty() { ++ return baseCoverage.isEmpty(); ++ } ++ public void intersection( Coverage other ) { ++ baseCoverage.intersection( other ); ++ } ++ public void union( Coverage other ) { ++ baseCoverage.union( other ); ++ } ++ public void extend( Object[] tuple ) { ++ baseCoverage.extend( unwrapTuple( tuple ) ); ++ } ++ public Supplier> createTestFactory() { ++ Predicate baseTest = ++ baseCoverage.createTestFactory().get(); ++ return () -> ( tuple -> baseTest.test( unwrapTuple( tuple ) ) ); ++ } ++ public String coverageText() { ++ return "WRONG UNITS[" + baseCoverage.coverageText() + "]"; ++ } ++ }; ++ } ++ + public double getScoreScale() { + return scoreWrapper_.wrapDouble( baseEngine_.getScoreScale() ); + } diff -Nru starjava-ttools-3.4.5/debian/patches/tjoin-remove-NdRange-and-all-references-to-it.patch starjava-ttools-3.4.6/debian/patches/tjoin-remove-NdRange-and-all-references-to-it.patch --- starjava-ttools-3.4.5/debian/patches/tjoin-remove-NdRange-and-all-references-to-it.patch 1970-01-01 00:00:00.000000000 +0000 +++ starjava-ttools-3.4.6/debian/patches/tjoin-remove-NdRange-and-all-references-to-it.patch 2022-07-14 17:35:39.000000000 +0000 @@ -0,0 +1,80 @@ +From: Mark Taylor +Date: Wed, 8 Jun 2022 17:52:28 +0100 +Subject: tjoin: remove NdRange and all references to it + +The rather ugly class NdRange, and the related MatchEngine method +getMatchBounds and canBoundMatch, are removed, since they are no +longer needed. This functionality has been replaced by the +Coverage abstraction. + +Backport from https://github.com/Starlink/starjava/commit/6d4e7fab +to allow newer tjoin package. +--- + .../ac/starlink/ttools/join/HumanMatchEngine.java | 40 ---------------------- + 1 file changed, 40 deletions(-) + +diff --git a/src/testcases/uk/ac/starlink/ttools/join/HumanMatchEngine.java b/src/testcases/uk/ac/starlink/ttools/join/HumanMatchEngine.java +index d465f4a..07aa1af 100644 +--- a/src/testcases/uk/ac/starlink/ttools/join/HumanMatchEngine.java ++++ b/src/testcases/uk/ac/starlink/ttools/join/HumanMatchEngine.java +@@ -7,7 +7,6 @@ import uk.ac.starlink.table.DescribedValue; + import uk.ac.starlink.table.ValueInfo; + import uk.ac.starlink.table.join.MatchEngine; + import uk.ac.starlink.table.join.MatchKit; +-import uk.ac.starlink.table.join.NdRange; + import uk.ac.starlink.ttools.func.CoordsRadians; + + /** +@@ -141,27 +140,6 @@ public class HumanMatchEngine implements MatchEngine { + return scoreInfo_; + } + +- public boolean canBoundMatch() { +- return baseEngine_.canBoundMatch(); +- } +- +- public NdRange getMatchBounds( NdRange[] inRanges, int index ) { +- int nr = inRanges.length; +- NdRange[] unwrappedInRanges = new NdRange[ nr ]; +- for ( int ir = 0; ir < nr; ir++ ) { +- NdRange inRng = inRanges[ ir ]; +- unwrappedInRanges[ ir ] = +- new NdRange( toComparables( unwrapTuple( inRng.getMins() ) ), +- toComparables( unwrapTuple( inRng.getMaxs() ) ) ); +- } +- NdRange unwrappedResult = +- baseEngine_.getMatchBounds( unwrappedInRanges, index ); +- return new NdRange( toComparables( wrapTuple( unwrappedResult +- .getMins() ) ), +- toComparables( wrapTuple( unwrappedResult +- .getMaxs() ) ) ); +- } +- + /** + * Unwraps a tuple of objects from a client of this engine, providing + * one suitable for the base engine. +@@ -192,24 +170,6 @@ public class HumanMatchEngine implements MatchEngine { + return wrapped; + } + +- /** +- * Converts an array of Object[] to an array of +- * Comparable[]. Anything which is not comparable is +- * replaced by null. +- * +- * @param objs objects +- * @return comparable array with the same contents as objs +- */ +- private Comparable[] toComparables( Object[] objs ) { +- Comparable[] comps = new Comparable[ objs.length ]; +- for ( int i = 0; i < comps.length; i++ ) { +- comps[ i ] = objs[ i ] instanceof Comparable +- ? (Comparable) objs[ i ] +- : null; +- } +- return comps; +- } +- + /** + * Creates a new ValueWrapper instance suitable for adapting values + * described by a given ValueInfo object. diff -Nru starjava-ttools-3.4.5/debian/patches/Use-a-local-copy-of-the-XSLT-stylesheets-instead-of-xdoc.patch starjava-ttools-3.4.6/debian/patches/Use-a-local-copy-of-the-XSLT-stylesheets-instead-of-xdoc.patch --- starjava-ttools-3.4.5/debian/patches/Use-a-local-copy-of-the-XSLT-stylesheets-instead-of-xdoc.patch 2022-06-13 14:59:22.000000000 +0000 +++ starjava-ttools-3.4.6/debian/patches/Use-a-local-copy-of-the-XSLT-stylesheets-instead-of-xdoc.patch 2022-07-14 17:35:39.000000000 +0000 @@ -10,7 +10,7 @@ 2 files changed, 17 insertions(+), 55 deletions(-) diff --git a/build.xml b/build.xml -index 6b8717d..944be7e 100644 +index 0fbd545..153dbae 100644 --- a/build.xml +++ b/build.xml @@ -114,7 +114,7 @@ diff -Nru starjava-ttools-3.4.5/debian/patches/Use-android-json.patch starjava-ttools-3.4.6/debian/patches/Use-android-json.patch --- starjava-ttools-3.4.5/debian/patches/Use-android-json.patch 2022-06-13 14:59:22.000000000 +0000 +++ starjava-ttools-3.4.6/debian/patches/Use-android-json.patch 2022-07-14 17:35:39.000000000 +0000 @@ -59,7 +59,7 @@ json.put( "nbformat", 4 ); json.put( "nbformat_minor", 2 ); diff --git a/src/main/uk/ac/starlink/ttools/build/Plot2Example.java b/src/main/uk/ac/starlink/ttools/build/Plot2Example.java -index 0978e7d..82de46f 100644 +index 9695c7c..06fb14c 100644 --- a/src/main/uk/ac/starlink/ttools/build/Plot2Example.java +++ b/src/main/uk/ac/starlink/ttools/build/Plot2Example.java @@ -31,6 +31,7 @@ import javax.swing.JComponent; diff -Nru starjava-ttools-3.4.5/debian/patches/Use-starlink-ttools.jar-as-main-jar-for-STILTS-and-add-to.patch starjava-ttools-3.4.6/debian/patches/Use-starlink-ttools.jar-as-main-jar-for-STILTS-and-add-to.patch --- starjava-ttools-3.4.5/debian/patches/Use-starlink-ttools.jar-as-main-jar-for-STILTS-and-add-to.patch 2022-06-13 14:59:22.000000000 +0000 +++ starjava-ttools-3.4.6/debian/patches/Use-starlink-ttools.jar-as-main-jar-for-STILTS-and-add-to.patch 2022-07-16 08:40:35.000000000 +0000 @@ -14,7 +14,7 @@ 1 file changed, 2 insertions(+) diff --git a/build.xml b/build.xml -index ff636b8..6b8717d 100644 +index 64deb83..0fbd545 100644 --- a/build.xml +++ b/build.xml @@ -246,6 +246,7 @@ diff -Nru starjava-ttools-3.4.5/.properties starjava-ttools-3.4.6/.properties --- starjava-ttools-3.4.5/.properties 2022-06-10 13:14:05.000000000 +0000 +++ starjava-ttools-3.4.6/.properties 2022-07-08 09:59:19.000000000 +0000 @@ -6,8 +6,6 @@ Moc.jar \ skyview-geom.jar \ jlatexmath.jar \ - ucidy-1.2-beta.jar \ - unity-1.1pre.jar \ jide-oss.jar \ ../util/util.jar \ ../table/table.jar \ @@ -25,6 +23,8 @@ ../vo/vo.jar \ ../pal/pal.jar \ ../dpac/dpac.jar \ + ../vo/ucidy-1.2-beta.jar \ + ../vo/unity-1.1pre.jar \ ../plastic/plastic.jar \ ../connect/connect.jar \ ../astrogrid/astrogrid.jar \ diff -Nru starjava-ttools-3.4.5/src/docs/sun256.xml starjava-ttools-3.4.6/src/docs/sun256.xml --- starjava-ttools-3.4.5/src/docs/sun256.xml 2022-06-10 13:14:05.000000000 +0000 +++ starjava-ttools-3.4.6/src/docs/sun256.xml 2022-07-08 09:59:19.000000000 +0000 @@ -76,6 +76,7 @@ + @@ -176,7 +177,7 @@ 256 -10 June 2022 +8 July 2022 STILTS web page: @@ -262,6 +263,7 @@ tcatn, tloop, tjoin, + arrayjoin, tgridmap, and tcube (see ). @@ -536,6 +538,7 @@ name of one of the tasks listed in - currently the available tasks are:
    +
  • arrayjoin
  • calc
  • cdsskymatch
  • cone
  • @@ -4277,6 +4280,25 @@ the requested Utype, the column value will be used.

    +
    Using value*() functions
    +

    You can use the special functions + valueDouble, valueInt, valueLong, + valueString and valueObject + to obtain the typed value of a column with a given name. + The argument of the function is a string giving the exact + (case-sensitive) column name, for instance + valueDouble("b_E(BP-RP)") + will yield the value of the column named "b_E(BP-RP)" + as a double-precision floating point value. + These functions are not the generally recommended way + to get column values, + since they are slower and provide less type-checking than + the other options listed above, + and can occasionally lead to some other esoteric problems. + However, if you need to refer by name to strangely-named columns + they are sometimes a convenient option. +

    +
    With the Object$ prefix

    If a column is referenced with the prefix "Object$" before its identifier @@ -5411,6 +5433,9 @@

  • tjoin: &tjoin-purpose;
  • +
  • arrayjoin: + &arrayjoin-purpose; +
  • tgridmap: &tgridmap-purpose;
  • @@ -5592,6 +5617,129 @@ a list of its command-line arguments, and some examples are given.

    + +<code>arrayjoin</code>: &arrayjoin-purpose; + +

    arrayjoin takes an input table and for each row +adds the contents of a separate "array" table. +The columns added are the columns from the array table, +and the value of each cell is the value of the whole column from +the array table represented as an array. +The assumption is that all the array tables have the same form +(the same columns, though not necessarily the same row counts). +

    + +

    This can be useful for constructing a single table +with array-valued columns containing data that is made +available in multiple external files, for instance +via the DataLink protocol; +this is illustrated in the Examples subsection below. +Note however that this command does not understand DataLink directly, +and cannot itself determine the location of the external array tables; +an expression giving their per-row location (filname or URL) +must be supplied. +

    + +&arrayjoin-summary; + + +Examples + +

    Here are some examples of using arrayjoin: +

    + +
    +stilts arrayjoin in=dr3-sources.vot + atable='"https://gea.esac.esa.int/data-server/data?RETRIEVAL_TYPE=XP_SAMPLED&RELEASE=Gaia+DR3&ID="+source_id' + icmd=progress + out=sources-with-xp.fits +
    +

    This command uses the Gaia DR3 DataLink service + to attach sampled XP spectrum data to an input table containing + other information about Gaia DR3 sources. +

    +

    The input file dr3-sources.vot must contain the column + source_id giving the Gaia DR3 source ID + corresponding to that row. + The atable expression is a URL built using that + source_id column and a base URL that can be obtained + by examining the DR3 catalogue + documentation, + or by examining the DataLink service descriptor returned from a + source catalogue query. +

    +

    The output file sources-with-xp.fits + is a FITS table with the same content as the input + but with three added array-valued columns, + wavelength, flux and flux_error + as supplied by tables downloaded from the URL in the atable + parameter. + For each row of the output table, the lengths of those three arrays + will be the same, namely the row count of the table that supplied them. + These matched arrays can be manipulated within STILTS, + for instance using the + Arrays functions + or to produce plots using the + lines or + arrayquantile plot layer types. +

    +

    The icmd=progress filter is a useful convenience to see + how the command is progressing, + since the operation requires multiple downloads and so may be + time-consuming. +

    +
    + +
    +stilts arrayjoin in=dl_dr3.fits + icmd='select has_epoch_photometry' + atable='"https://gea.esac.esa.int/data-server/data?RETRIEVAL_TYPE=EPOCH_PHOTOMETRY&ID=Gaia+DR3+"+source_id' \ + afmt=votable \ + acmd='select equals(BAND,\"G\")' \ + acmd='keepcols "time mag flux"' \ + out=with-epoch-photom.fits +
    +

    Similar to the previous example, but this time acquiring + EPOCH_PHOTOMETRY rather than XP_SAMPLED data, + and performing some additional + processing of the main and array input tables. + Only those input rows are used for which the boolean + has_epoch_photometry column is True; + for the linked array tables the rows are limited to those with + the BAND column having the value "G"; + and only the array columns + time, mag and flux + are retained for inclusion into the output. +

    + +
    +stilts arrayjoin in=obscore-lotssdr2.vot + atable='equals(dataproduct_type,"spectrum")?access_url:""' + out=with-spectra.vot +
    +

    Takes an ivoa.ObsCore-like table and adds array-valued + columns giving the content of spectrum files linked from + the access_url field. + For rows in the input table whose dataproduct_type + field does not have the value "spectrum" a blank location + is substituted insteead of the access_url. + That has the effect of ensuring that no attempt is made + to add array values for non-spectrum rows of the input table, + while retaining those rows in the output. + Note all the access_urls in the selected rows must point to + spectrum files of a of a similar form, + otherwise the command will fail. +

    + +
    +

    + +
    + +
    + + <code>calc</code>: &calc-purpose; @@ -13876,6 +14024,38 @@

+

+ +
Version 3.4-6 (8 July 2022)
+

+

+
Enhancements
+

+

    +
  • New command arrayjoin + added.
  • +
  • Modify functions add, subtract, + multiply, divide in class + Arrays; + these now all take either two array arguments or an array + and a scalar in either order.
  • +
  • New function dotProduct in class + Arrays.
  • +
  • Provide value* functions for + column references + to strangely-named columns.
  • +
  • Minor taplint EPN stage + update; now corresponds to PR-EPNTAP-20220705.
  • +
+

+
Bug Fix
+

+

    +
  • Fix FITS output so it doesn't fail when attempting to write + metadata with non-ASCII Unicode characters.
  • +
+

+

diff -Nru starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/build/Plot2Example.java starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/build/Plot2Example.java --- starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/build/Plot2Example.java 2022-06-10 13:14:05.000000000 +0000 +++ starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/build/Plot2Example.java 2022-07-08 09:59:19.000000000 +0000 @@ -1655,7 +1655,7 @@ "*in=" + TName.LRS, null, "*shading=aux", "*aux=epoch", null, - "*xs=arrayFunc(\"2.998e8/x\",param$Frequency)", + "*xs=divide(2.998e8,param$Frequency)", "*ys=multiply(add(RX1,RX2),0.5)", null, "layer_l=lines", "thick_l=2", null, "*layer_e=yerrors", diff -Nru starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/func/Arrays.java starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/func/Arrays.java --- starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/func/Arrays.java 2022-06-10 13:14:05.000000000 +0000 +++ starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/func/Arrays.java 2022-07-08 09:59:19.000000000 +0000 @@ -10,6 +10,8 @@ import java.lang.reflect.Array; import java.util.HashMap; import java.util.Map; +import java.util.function.DoubleBinaryOperator; +import java.util.function.IntToDoubleFunction; import java.util.logging.Level; import java.util.logging.Logger; import uk.ac.starlink.table.Tables; @@ -84,6 +86,11 @@ @HideDoc public static final String ARRAY_INDEX_VARNAME = "i"; + private static final DoubleBinaryOperator ADD = (a, b) -> a + b; + private static final DoubleBinaryOperator SUBTRACT = (a, b) -> a - b; + private static final DoubleBinaryOperator MULTIPLY = (a, b) -> a * b; + private static final DoubleBinaryOperator DIVIDE = (a, b) -> a / b; + private static final ThreadLocal afuncsThreadLocal_ = ThreadLocal.withInitial( ArrayFuncMap::new ); private static final Logger logger_ = @@ -359,183 +366,124 @@ } /** - * Returns the result of adding two numeric arrays element by element. - * Both arrays must be numeric, and the arrays must have the same length. - * If either of those conditions is not true, null is returned. - * The types of the arrays do not need to be the same, - * so for example it is permitted to add an integer array - * to a floating point array. - * - * @example add(array(1,2,3), array(0.1,0.2,0.3)) - * = [1.1, 2.2, 3.3] - * - * @param array1 first array of numeric values - * @param array2 second array of numeric values - * @return element-by-element sum of - * array1 and array2, - * the same length as the input arrays - */ - public static double[] add( Object array1, Object array2 ) { - int n = getNumericArrayLength( array1 ); - if ( n >= 0 && getNumericArrayLength( array2 ) == n ) { - double[] out = new double[ n ]; - for ( int i = 0; i < n; i++ ) { - out[ i ] = Array.getDouble( array1, i ) - + Array.getDouble( array2, i ); + * Returns the dot (scalar) product of two numeric arrays. + * If either argument is not an array, or if the arrays are not of + * the same length, a blank value is returned. + * + * @example dotProduct(array(3,4,5), array(1,2,3)) = 26 + * + * @param array1 first array + * @param array2 second array + * @return sum of element-wise products of input arrays + */ + public static double dotProduct( Object array1, Object array2 ) { + int n1 = getNumericArrayLength( array1 ); + int n2 = getNumericArrayLength( array2 ); + if ( n1 >= 0 && n1 == n2 ) { + double d = 0; + for ( int i = 0; i < n1; i++ ) { + d += Array.getDouble( array1, i ) + * Array.getDouble( array2, i ); } - return out; + return d; } else { - return null; + return Double.NaN; } } /** - * Returns the result of adding a constant value to every element of - * a numeric array. - * If the supplied array argument is not a numeric array, - * null is returned. + * Returns the element-by-element result of adding + * either two numeric arrays of the same length, + * or an array and a scalar considered as if an array of the right length. + * + *

If the arguments are not as expected + * (e.g. arrays of different lengths, both scalars, not numeric) + * then null is returned. * + * @example add(array(1,2,3), array(0.1,0.2,0.3)) + * = [1.1, 2.2, 3.3] * @example add(array(1,2,3), 10) = [11,12,13] * - * @param array array input - * @param constant value to add to each array element - * @return array output, - * the same length as the array parameter + * @param arrayOrScalar1 first numeric array/scalar + * @param arrayOrScalar2 second numeric array/scalar + * @return element-by-element result of + * arrayOrScalar1 + arrayOrScalar2, + * the same length as the input array(s) */ - public static double[] add( Object array, double constant ) { - int n = getNumericArrayLength( array ); - if ( n >= 0 ) { - double[] out = new double[ n ]; - for ( int i = 0; i < n; i++ ) { - out[ i ] = Array.getDouble( array, i ) + constant; - } - return out; - } - else { - return null; - } + public static double[] add( Object arrayOrScalar1, Object arrayOrScalar2 ) { + return arrayOp( arrayOrScalar1, arrayOrScalar2, ADD ); } /** - * Returns the result of subtracting one numeric array from the other - * element by element. - * Both arrays must be numeric, and the arrays must have the same length. - * If either of those conditions is not true, null is returned. - * The types of the arrays do not need to be the same, - * so for example it is permitted to subtract an integer array - * from a floating point array. + * Returns the element-by-element result of subtracting + * either two numeric arrays of the same length, + * or an array and a scalar considered as if an array of the right length. + * + *

If the arguments are not as expected + * (e.g. arrays of different lengths, both scalars, not numeric) + * then null is returned. * * @example subtract(array(1,2,3), array(0.1,0.2,0.3)) * = [0.9, 1.8, 2.7] + * @example subtract(array(1,2,3), 1.0) + * = [0, 1, 2] * - * @param array1 first array of numeric values - * @param array2 second array of numeric values - * @return element-by-element difference of - * array1 and array2, - * the same length as the input arrays - */ - public static double[] subtract( Object array1, Object array2 ) { - int n = getNumericArrayLength( array1 ); - if ( n >= 0 && getNumericArrayLength( array2 ) == n ) { - double[] out = new double[ n ]; - for ( int i = 0; i < n; i++ ) { - out[ i ] = Array.getDouble( array1, i ) - - Array.getDouble( array2, i ); - } - return out; - } - else { - return null; - } + * @param arrayOrScalar1 first numeric array/scalar + * @param arrayOrScalar2 second numeric array/scalar + * @return element-by-element result of + * arrayOrScalar1 - arrayOrScalar2, + * the same length as the input array(s) + */ + public static double[] subtract( Object arrayOrScalar1, + Object arrayOrScalar2 ) { + return arrayOp( arrayOrScalar1, arrayOrScalar2, SUBTRACT ); } /** - * Returns the result of multiplying two numeric arrays element by element. - * Both arrays must be numeric, and the arrays must have the same length. - * If either of those conditions is not true, null is returned. - * The types of the arrays do not need to be the same, - * so for example it is permitted to multiply an integer array - * by a floating point array. + * Returns the element-by-element result of multiplying + * either two numeric arrays of the same length, + * or an array and a scalar considered as if an array of the right length. * - * @example multiply(array(1,2,3), array(2,4,6)) = [2, 8, 18] - * - * @param array1 first array of numeric values - * @param array2 second array of numeric values - * @return element-by-element product of - * array1 and array2, - * the same length as the input arrays - */ - public static double[] multiply( Object array1, Object array2 ) { - int n = getNumericArrayLength( array1 ); - if ( n >= 0 && getNumericArrayLength( array2 ) == n ) { - double[] out = new double[ n ]; - for ( int i = 0; i < n; i++ ) { - out[ i ] = Array.getDouble( array1, i ) - * Array.getDouble( array2, i ); - } - return out; - } - else { - return null; - } - } - - /** - * Returns the result of multiplying every element of a numeric array - * by a constant value. - * If the supplied array argument is not a numeric array, - * null is returned. + *

If the arguments are not as expected + * (e.g. arrays of different lengths, both scalars, not numeric) + * then null is returned. * - * @example multiply(array(1,2,3), 2) = [2, 4, 6] + * @example multiply(array(1,2,3), array(2,4,6)) = [2, 8, 18] + * @example multiply(2, array(1,2,3)) = [2, 4, 6] * - * @param array array input - * @param constant value by which to multiply each array element - * @return array output, - * the same length as the array parameter + * @param arrayOrScalar1 first numeric array/scalar + * @param arrayOrScalar2 second numeric array/scalar + * @return element-by-element result of + * arrayOrScalar1 * arrayOrScalar2, + * the same length as the input array(s) */ - public static double[] multiply( Object array, double constant ) { - int n = getNumericArrayLength( array ); - if ( n >= 0 ) { - double[] out = new double[ n ]; - for ( int i = 0; i < n; i++ ) { - out[ i ] = Array.getDouble( array, i ) * constant; - } - return out; - } - else { - return null; - } + public static double[] multiply( Object arrayOrScalar1, + Object arrayOrScalar2 ) { + return arrayOp( arrayOrScalar1, arrayOrScalar2, MULTIPLY ); } /** - * Returns the result of dividing two numeric arrays element by element. - * Both arrays must be numeric, and the arrays must have the same length. - * If either of those conditions is not true, null is returned. - * The types of the arrays do not need to be the same, - * so for example it is permitted to divide an integer array - * by a floating point array. + * Returns the element-by-element result of dividing + * either two numeric arrays of the same length, + * or an array and a scalar considered as if an array of the right length. + * + *

If the arguments are not as expected + * (e.g. arrays of different lengths, both scalars, not numeric) + * then null is returned. * * @example divide(array(0,9,4), array(1,3,8)) = [0, 3, 0.5] + * @example divide(array(50,60,70), 10) = [5, 6, 7] * - * @param array1 array of numerator values (numeric) - * @param array2 array of denominator values (numeric) - * @return element-by-element result of array1[i]/array2[i] - * the same length as the input arrays - */ - public static double[] divide( Object array1, Object array2 ) { - int n = getNumericArrayLength( array1 ); - if ( n >= 0 && getNumericArrayLength( array2 ) == n ) { - double[] out = new double[ n ]; - for ( int i = 0; i < n; i++ ) { - out[ i ] = Array.getDouble( array1, i ) - / Array.getDouble( array2, i ); - } - return out; - } - else { - return null; - } + * @param arrayOrScalar1 first numeric array/scalar + * @param arrayOrScalar2 second numeric array/scalar + * @return element-by-element result of + * arrayOrScalar1 / arrayOrScalar2, + * the same length as the input array(s) + */ + public static double[] divide( Object arrayOrScalar1, + Object arrayOrScalar2 ) { + return arrayOp( arrayOrScalar1, arrayOrScalar2, DIVIDE ); } /** @@ -1319,6 +1267,129 @@ } /** + * Performs an operation on two input numeric arrays returning one + * numeric array of the same length. + * One or other, but not both, of the input array values may in fact + * be a scalar which is treated as an array of the appropriate length. + * + * @param aos1 first array or scalar + * @param aos2 second array or scalar + * @param operator element-wise operation + * @return array result of applying operator to each pair of elements, + * or null if input operands are not suitable + */ + private static double[] arrayOp( Object aos1, Object aos2, + DoubleBinaryOperator operator ) { + ElementHandler eh1 = createElementHandler( aos1 ); + if ( eh1 == null ) { + return null; + } + ElementHandler eh2 = createElementHandler( aos2 ); + if ( eh2 == null ) { + return null; + } + int leng1 = eh1.length_; + int leng2 = eh2.length_; + final int leng; + if ( leng1 == leng2 ) { + leng = leng1; + } + else if ( leng1 < 0 ) { + leng = leng2; + } + else if ( leng2 < 0 ) { + leng = leng1; + } + else { + leng = -1; + } + if ( leng >= 0 ) { + double[] result = new double[ leng ]; + for ( int i = 0; i < leng; i++ ) { + result[ i ] = operator.applyAsDouble( eh1.getElement( i ), + eh2.getElement( i ) ); + } + return result; + } + else { + return null; + } + } + + /** + * Constructs an ElementHandler from an argument that may be either + * a numeric (wrapper) scalar or a numeric (primitive) array. + * In case of a scalar, the length is reported as -1. + * + * @param aos array or scalar + * @return element handler, or null if input is not suitable + */ + private static ElementHandler createElementHandler( Object aos ) { + if ( aos instanceof Number ) { + final double scalar = ((Number) aos).doubleValue(); + return new ElementHandler( i -> scalar, -1 ); + } + else if ( aos instanceof double[] ) { + final double[] array = (double[]) aos; + return new ElementHandler( i -> array[ i ], array.length ); + } + else if ( aos instanceof float[] ) { + final float[] array = (float[]) aos; + return new ElementHandler( i -> array[ i ], array.length ); + } + else if ( aos instanceof int[] ) { + final int[] array = (int[]) aos; + return new ElementHandler( i -> array[ i ], array.length ); + } + else if ( aos instanceof short[] ) { + final short[] array = (short[]) aos; + return new ElementHandler( i -> array[ i ], array.length ); + } + else if ( aos instanceof long[] ) { + final long[] array = (long[]) aos; + return new ElementHandler( i -> array[ i ], array.length ); + } + else if ( aos instanceof byte[] ) { + final byte[] array = (byte[]) aos; + return new ElementHandler( i -> array[ i ], array.length ); + } + else { + return null; + } + } + + /** + * Provides indexed double values. + * This serves as a generalised representation of a double array. + */ + private static class ElementHandler { + final int length_; + final IntToDoubleFunction accessor_; + + /** + * Constructs an ElementHandler with a given finite length. + * + * @param accessor provides value access + * @param length number of elements available; + * if negative any input index is acceptable + */ + ElementHandler( IntToDoubleFunction accessor, int length ) { + accessor_ = accessor; + length_ = length; + } + + /** + * Returns the value at a given index. + * + * @param i index + * @return value at index + */ + double getElement( int i ) { + return accessor_.applyAsDouble( i ); + } + } + + /** * Class to cache compiled array functions. */ private static class ArrayFuncMap { diff -Nru starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/func/VO.java starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/func/VO.java --- starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/func/VO.java 2022-06-10 13:14:05.000000000 +0000 +++ starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/func/VO.java 2022-07-08 09:59:19.000000000 +0000 @@ -5,8 +5,8 @@ package uk.ac.starlink.ttools.func; -import uk.ac.starlink.ttools.votlint.UcdStatus; -import uk.ac.starlink.ttools.votlint.UnitStatus; +import uk.ac.starlink.vo.UcdStatus; +import uk.ac.starlink.vo.UnitStatus; /** * Virtual Observatory-specific functions. diff -Nru starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/jel/StarTableJELRowReader.java starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/jel/StarTableJELRowReader.java --- starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/jel/StarTableJELRowReader.java 2022-06-10 13:14:05.000000000 +0000 +++ starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/jel/StarTableJELRowReader.java 2022-07-08 09:59:19.000000000 +0000 @@ -1,7 +1,9 @@ package uk.ac.starlink.ttools.jel; import java.io.IOException; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Logger; @@ -57,6 +59,21 @@ * within this reader. The quality of the random numbers may not * be particularly good. * + *

"value*()" functions: + *
The methods {@link #valueDouble valueDouble}, + * {@link #valueInt valueInt}, {@link #valueLong valueLong}, + * {@link #valueString valueString} and {@link #valueObject valueObject} + * are provided. + * Each takes as an argument an exact column name, and provides the + * typed value of the column at the current row. + * These methods are not the generally recommended way to + * access column values; they are slower and less tidy than simply + * using column names or $IDs in expressions, and do not admit of + * static analysis, so their use can not be reflected in + * the results of {@link #getTranslatedColumns getTranslatedColumns}. + * However, it does allow access by column name to columns with names + * that are not legal java identifiers. + * * * * @author Mark Taylor @@ -65,6 +82,7 @@ public abstract class StarTableJELRowReader extends JELRowReader { private final StarTable table_; + private Map colMetaMap_; private static final AtomicInteger seeder_ = new AtomicInteger(); private static final Logger logger_ = Logger.getLogger( "uk.ac.starlink.ttools.jel" ); @@ -150,6 +168,116 @@ return false; } + /** + * Returns the value of a named column in this reader's table + * at the current row as a double. + * This is not generally the recommended way to access column values, + * but it will work for column names without any syntactical restrictions. + * + * @param colName column name, matched exactly + * @return value of named column as a double, + * or NaN on failure (no such column or value not numeric) + */ + public double valueDouble( String colName ) { + ColMeta colMeta = getColMetaByExactName( colName ); + if ( colMeta != null && colMeta.isNumber_ ) { + Object value = getCellValue( colMeta.icol_ ); + if ( value instanceof Number ) { + return ((Number) value).doubleValue(); + } + } + return Double.NaN; + } + + /** + * Returns the value of a named column in this reader's table + * at the current row as an int. + * This is not generally the recommended way to access column values, + * but it will work for column names without any syntactical restrictions. + * + * @param colName column name, matched exactly + * @return value of column as an int; + * foundNull is called on failure + * (no such column or value not numeric or not finite) + */ + public int valueInt( String colName ) { + ColMeta colMeta = getColMetaByExactName( colName ); + if ( colMeta != null && colMeta.isNumber_ ) { + Object value = getCellValue( colMeta.icol_ ); + if ( value instanceof Number ) { + Number num = (Number) value; + if ( Double.isFinite( num.doubleValue() ) ) { + return num.intValue(); + } + } + } + foundNull(); + return 0; + } + + /** + * Returns the value of a named column in this reader's table + * at the current row as a long int. + * This is not generally the recommended way to access column values, + * but it will work for column names without any syntactical restrictions. + * + * @param colName column name, matched exactly + * @return value of column as a long; + * foundNull is called on failure + * (no such column or value not numeric or not finite) + */ + public long valueLong( String colName ) { + ColMeta colMeta = getColMetaByExactName( colName ); + if ( colMeta != null && colMeta.isNumber_ ) { + Object value = getCellValue( colMeta.icol_ ); + if ( value instanceof Number ) { + Number num = (Number) value; + if ( Double.isFinite( num.doubleValue() ) ) { + return num.longValue(); + } + } + } + foundNull(); + return 0L; + } + + /** + * Returns the value of a named column in this reader's table + * at the current row as a String. + * This is not generally the recommended way to access column values, + * but it will work for column names without any syntactical restrictions. + * + * @param colName column name, matched exactly + * @return value of column as a String, or null if no such column + */ + public String valueString( String colName ) { + ColMeta colMeta = getColMetaByExactName( colName ); + if ( colMeta != null ) { + Object value = getCellValue( colMeta.icol_ ); + if ( value != null ) { + return value.toString(); + } + } + return null; + } + + /** + * Returns the value of a named column in this reader's table + * at the current row as an Object; + * This is not generally the recommended way to access column values, + * but it will work for column names without any syntactical restrictions. + * + * @param colName column name, matched exactly + * @return value of column as an Object, or null if no such column + */ + public Object valueObject( String colName ) { + ColMeta colMeta = getColMetaByExactName( colName ); + if ( colMeta != null ) { + return getCellValue( colMeta.icol_ ); + } + return null; + } + protected boolean isBlank( int icol ) { try { return Tables.isBlank( getCell( icol ) ); @@ -442,6 +570,35 @@ } /** + * Returns a ColMeta object corresponding to the supplied column name. + * There are no lexical restrictions on the form of the name, + * and matching is exact. + * + * @param name column name + * @return metadata for column, + * or null if no column with the given name exists + */ + private ColMeta getColMetaByExactName( String name ) { + + /* Construct the required map lazily, since for most instances + * of this class it will never be used. + * This code is effectively thread-safe; the worst that can happen + * is that N identical instances will be constructed concurrently + * and N-1 will be discarded. */ + if ( colMetaMap_ == null ) { + Map map = new HashMap<>(); + int ncol = table_.getColumnCount(); + for ( int icol = 0; icol < ncol; icol++ ) { + ColumnInfo info = table_.getColumnInfo( icol ); + map.put( info.getName(), + new ColMeta( icol, info.getContentClass() ) ); + } + colMetaMap_ = map; + } + return colMetaMap_.get( name ); + } + + /** * Takes a (non-prefixed) UCD specification and returns a Pattern * actual UCDs should match if they represent the same thing. * Punctuation is mapped to underscores, and the pattern is @@ -474,4 +631,25 @@ String regex = utype.replaceAll( "_", "\\\\W" ); return Pattern.compile( regex, Pattern.CASE_INSENSITIVE ); } + + /** + * Aggregates a column index and class information for a table column. + */ + private static class ColMeta { + final int icol_; + final boolean isNumber_; + final boolean isString_; + + /** + * Constructor. + * + * @param icol column index in table + * @param clazz content class of column + */ + ColMeta( int icol, Class clazz ) { + icol_ = icol; + isNumber_ = Number.class.isAssignableFrom( clazz ); + isString_ = String.class.equals( clazz ); + } + } } diff -Nru starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/Stilts.java starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/Stilts.java --- starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/Stilts.java 2022-06-10 13:14:05.000000000 +0000 +++ starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/Stilts.java 2022-07-08 09:59:19.000000000 +0000 @@ -146,6 +146,7 @@ /* Populate main task factory with non-plot tasks. */ plot2TaskFactory_ = p2fact; String taskPkg = "uk.ac.starlink.ttools.task."; + taskFactory_.register( "arrayjoin", taskPkg + "ArrayJoin" ); taskFactory_.register( "calc", taskPkg + "Calc" ); taskFactory_.register( "cdsskymatch", taskPkg + "CdsUploadSkyMatch" ); taskFactory_.register( "cone", taskPkg + "TableCone" ); diff -Nru starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/taplint/EpnTapStage.java starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/taplint/EpnTapStage.java --- starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/taplint/EpnTapStage.java 2022-06-10 13:14:05.000000000 +0000 +++ starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/taplint/EpnTapStage.java 2022-07-08 09:59:19.000000000 +0000 @@ -20,13 +20,13 @@ import uk.ac.starlink.table.Tables; import uk.ac.starlink.table.ValueInfo; import uk.ac.starlink.ttools.func.Times; -import uk.ac.starlink.ttools.votlint.UcdStatus; import uk.ac.starlink.ttools.votlint.VocabChecker; import uk.ac.starlink.vo.ColumnMeta; import uk.ac.starlink.vo.SchemaMeta; import uk.ac.starlink.vo.TableMeta; import uk.ac.starlink.vo.TapQuery; import uk.ac.starlink.vo.TapService; +import uk.ac.starlink.vo.UcdStatus; import uk.ac.starlink.vo.VocabTerm; /** @@ -804,7 +804,7 @@ textCol( "bib_reference", "meta.bib" ), textCol( "internal_reference", "meta.id.cross" ), textCol( "external_link", "meta.ref.url" ), - new SingleCol( "shape", Type.TEXT_MOC, null, + new SingleCol( "coverage", Type.TEXT_MOC, null, "pos.outline;obs.field" ), textCol( "spatial_coordinate_description", "meta.code.class;pos.frame" ), @@ -856,7 +856,7 @@ FixedCode.W_PNTS, FixedCode.W_VCPD ); // Currently not tested but could be: compliant MOC strings in - // shape column. + // coverage column. return colMap.values().toArray( new EpnCol[ 0 ] ); } diff -Nru starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/taplint/UnitUcdStage.java starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/taplint/UnitUcdStage.java --- starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/taplint/UnitUcdStage.java 2022-06-10 13:14:05.000000000 +0000 +++ starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/taplint/UnitUcdStage.java 2022-07-08 09:59:19.000000000 +0000 @@ -8,12 +8,12 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; -import uk.ac.starlink.ttools.votlint.UcdStatus; -import uk.ac.starlink.ttools.votlint.UnitStatus; import uk.ac.starlink.vo.ColumnMeta; import uk.ac.starlink.vo.SchemaMeta; import uk.ac.starlink.vo.TableMeta; import uk.ac.starlink.vo.TapService; +import uk.ac.starlink.vo.UcdStatus; +import uk.ac.starlink.vo.UnitStatus; /** * Validation stage for checking column metadata elements from diff -Nru starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/task/AbstractInputTableParameter.java starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/task/AbstractInputTableParameter.java --- starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/task/AbstractInputTableParameter.java 2022-06-10 13:14:05.000000000 +0000 +++ starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/task/AbstractInputTableParameter.java 2022-07-08 09:59:19.000000000 +0000 @@ -135,22 +135,7 @@ boolean stream = getStreamParameter().booleanValue( env ); StarTableFactory tfact = LineTableEnvironment.getTableFactory( env ); try { - if ( loc.equals( "-" ) ) { - InputStream in = - new BufferedInputStream( DataSource - .getInputStream( loc, - allowSystem_ ) ); - return getStreamedTable( tfact, in, fmt, null ); - } - else if ( stream ) { - return getStreamedTable( tfact, - DataSource - .makeDataSource( loc, allowSystem_ ), - fmt ); - } - else { - return tfact.makeStarTable( loc, fmt ); - } + return makeTable( loc, fmt, stream, tfact ); } catch ( EOFException e ) { throw new ExecutionException( "Premature end of file", e ); @@ -164,6 +149,35 @@ } } + /** + * Reads a table given fixed values for the various parameters. + * + * @param loc table location + * @param fmt input format string + * @param stream true for streamed input + * @param tfact table factory + * @return table loaded + */ + public StarTable makeTable( String loc, String fmt, boolean stream, + StarTableFactory tfact ) + throws IOException, TaskException { + if ( loc.equals( "-" ) ) { + InputStream in = + new BufferedInputStream( DataSource + .getInputStream( loc, allowSystem_ ) ); + return getStreamedTable( tfact, in, fmt, null ); + } + else if ( stream ) { + return getStreamedTable( tfact, + DataSource + .makeDataSource( loc, allowSystem_ ), + fmt ); + } + else { + return tfact.makeStarTable( loc, fmt ); + } + } + /** * Constructs an array of tables from a location string given the current * state of this parameter and its associated parameter values. diff -Nru starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/task/ArrayJoin.java starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/task/ArrayJoin.java --- starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/task/ArrayJoin.java 1970-01-01 00:00:00.000000000 +0000 +++ starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/task/ArrayJoin.java 2022-07-08 09:59:19.000000000 +0000 @@ -0,0 +1,788 @@ +package uk.ac.starlink.ttools.task; + +import gnu.jel.CompilationException; +import gnu.jel.CompiledExpression; +import gnu.jel.Library; +import java.io.IOException; +import java.lang.reflect.Array; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.function.Function; +import java.util.logging.Logger; +import uk.ac.starlink.table.AbstractStarTable; +import uk.ac.starlink.table.ColumnInfo; +import uk.ac.starlink.table.DescribedValue; +import uk.ac.starlink.table.JoinFixAction; +import uk.ac.starlink.table.JoinStarTable; +import uk.ac.starlink.table.RowSequence; +import uk.ac.starlink.table.RowSplittable; +import uk.ac.starlink.table.RowStore; +import uk.ac.starlink.table.StarTable; +import uk.ac.starlink.table.StarTableFactory; +import uk.ac.starlink.table.StoragePolicy; +import uk.ac.starlink.table.Tables; +import uk.ac.starlink.table.ValueInfo; +import uk.ac.starlink.table.WrapperStarTable; +import uk.ac.starlink.task.BooleanParameter; +import uk.ac.starlink.task.Environment; +import uk.ac.starlink.task.ExecutionException; +import uk.ac.starlink.task.Parameter; +import uk.ac.starlink.task.ParameterValueException; +import uk.ac.starlink.task.StringParameter; +import uk.ac.starlink.task.TaskException; +import uk.ac.starlink.ttools.filter.ProcessingStep; +import uk.ac.starlink.ttools.jel.JELRowReader; +import uk.ac.starlink.ttools.jel.JELUtils; +import uk.ac.starlink.ttools.jel.SequentialJELRowReader; +import uk.ac.starlink.util.IOFunction; + +/** + * Task to add the contents of an external table for each row of + * an input table as array-valued columns. + * + * @author Mark Taylor + * @since 17 Jun 2022 + */ +public class ArrayJoin extends SingleMapperTask { + + private final ExpressionInputTableParameter atableParam_; + private final InputFormatParameter afmtParam_; + private final FilterParameter acmdParam_; + private final BooleanParameter astreamParam_; + private final BooleanParameter cacheParam_; + private final JoinFixActionParameter fixcolsParam_; + private final StringParameter asuffixParam_; + + private static final Logger logger_ = + Logger.getLogger( "uk.ac.starlink.ttools.task" ); + + /** + * Constructor. + */ + public ArrayJoin() { + super( "Adds table-per-row data as array-valued columns", + new ChoiceMode(), true, true ); + + acmdParam_ = new FilterParameter( "acmd" ); + + atableParam_ = new ExpressionInputTableParameter( "atable" ); + atableParam_.setTableDescription( "array tables" ); + atableParam_.setPrompt( "Per-row location of tables with array data" ); + atableParam_.setUsage( "" ); + atableParam_.setDescription( new String[] { + "

Gives the location of the table whose rows will be turned into", + "an array-valued column.", + "This will generally be an expression", + "giving a URL or filename that is different", + "for each row of the input table.", + "If table loading fails for the given location,", + "for instance becase the file is not found or an HTTP 404", + "response is received,", + "the array cells in the corresponding row will be blank.", + "

", + "

The first non-blank table loaded defines the array columns", + "to be added.", + "If subsequent tables have a different structure", + "(do not contain similar columns in a similar sequence)", + "an error may result.", + "If the external array tables are not all homogenous in this way,", + "the " + acmdParam_.getName() + " parameter", + "can be used to filter them so that they are.", + "

", + } ); + afmtParam_ = atableParam_.getFormatParameter(); + astreamParam_ = atableParam_.getStreamParameter(); + acmdParam_.setTableDescription( "array tables", atableParam_, true ); + + cacheParam_ = new BooleanParameter( "cache" ); + cacheParam_.setPrompt( "Cache array values?" ); + cacheParam_.setBooleanDefault( true ); + cacheParam_.setDescription( new String[] { + "

Determines whether the array data will be cached", + "the first time an array table is read (true)", + "or re-read from the array table every time", + "the row is accessed (false).", + "Since the row construction may be an expensive step,", + "especially if the tables are downloaded,", + "it usually makes sense to set this true (the default).", + "When true it also enables the metadata to be adjusted", + "to report constant array length where applicable,", + "which cannot be done before all the rows have been scanned,", + "and which may enable more efficient file output.", + "However, if you want to stream the data you can set it false.", + "

", + } ); + + fixcolsParam_ = new JoinFixActionParameter( "fixcols" ); + asuffixParam_ = + fixcolsParam_ + .createSuffixParameter( "suffixarray", "the array tables", "_a" ); + } + + @Override + public Parameter[] getParameters() { + List> params = new ArrayList<>(); + params.addAll( Arrays.asList( super.getParameters() ) ); + params.addAll( Arrays.asList( new Parameter[] { + atableParam_, + afmtParam_, + astreamParam_, + acmdParam_, + cacheParam_, + fixcolsParam_, + asuffixParam_, + } ) ); + return params.toArray( new Parameter[ 0 ] ); + } + + public TableProducer createProducer( Environment env ) + throws TaskException { + final TableProducer inProd = createInputProducer( env ); + final String atableExpr = atableParam_.stringValue( env ); + String fmt = afmtParam_.stringValue( env ); + boolean stream = astreamParam_.booleanValue( env ); + final ProcessingStep[] asteps = acmdParam_.stepsValue( env ); + StarTableFactory tfact = LineTableEnvironment.getTableFactory( env); + boolean allowMissing = true; + boolean isCached = cacheParam_.booleanValue( env ); + final IOFunction atableReader = loc -> { + logger_.info( "Loading table: " + loc ); + StarTable atable; + try { + atable = atableParam_.makeTable( loc, fmt, stream, tfact ); + } + catch ( IOException e ) { + if ( allowMissing ) { + return null; + } + else { + throw e; + } + } + catch ( TaskException e ) { + throw new IOException( e.getMessage(), e ); + } + InputTableSpec spec = + InputTableSpec.createSpec( loc, asteps, atable ); + try { + return spec.getWrappedTable(); + } + catch ( TaskException e ) { + throw new IOException( "Trouble filtering array table " + loc, + e ); + } + }; + JoinFixAction inFixact = JoinFixAction.NO_ACTION; + JoinFixAction aFixact = + fixcolsParam_.getJoinFixAction( env, asuffixParam_ ); + return new TableProducer() { + public StarTable getTable() throws IOException, TaskException { + final StarTable inTable = inProd.getTable(); + final Function alocCompiler; + try { + alocCompiler = + JELUtils.compiler( inTable, atableExpr, String.class ); + } + catch ( CompilationException e ) { + throw new ParameterValueException( atableParam_, + "Bad table location", + e ); + } + ArrayDataTable seqArrayTable = + createSequentialArrayTable( inTable, alocCompiler, + atableReader ); + StarTable arrayTable = isCached + ? cacheArrayTable( seqArrayTable ) + : seqArrayTable; + StarTable[] tables = { inTable, arrayTable }; + JoinFixAction[] fixacts = { inFixact, aFixact }; + return new JoinStarTable( tables, fixacts ); + } + }; + } + + /** + * Generates the joined table given required information. + * + * @param inTable input table + * @param alocCompiler produces location of array table + * for input table row + * @param atableLoader loads an array table given a location + * @return non-random table joining input and arrays + */ + private static ArrayDataTable + createSequentialArrayTable( + StarTable inTable, + Function alocCompiler, + IOFunction atableLoader ) + throws IOException, TaskException { + + /* Read through the input table to find the first array table. + * We will use this as a template supplying the array column + * metadata. */ + SequentialJELRowReader inRdr0 = new SequentialJELRowReader( inTable ); + Library lib = JELUtils.getLibrary( inRdr0 ); + CompiledExpression alocCompex = alocCompiler.apply( lib ); + int[] rowIndex = new int[] { -1 }; + StarTable atable0 = readFirstArrayTable( inTable, alocCompiler, + atableLoader, rowIndex); + if ( atable0 == null ) { + throw new TaskException( "No array tables" ); + } + int irow0 = rowIndex[ 0 ]; + + /* Prepare typed array handlers for the array columns we + * have discovered. */ + int nacol = atable0.getColumnCount(); + List> acolList = new ArrayList<>( nacol ); + for ( int ic = 0; ic < nacol; ic++ ) { + ColumnInfo cinfo = atable0.getColumnInfo( ic ); + ArrayColumn acol = createArrayColumn( cinfo, ic ); + if ( acol != null ) { + acolList.add( acol ); + } + else { + logger_.warning( "Array storage not supported for column " + + cinfo + " - ignoring" ); + } + } + ArrayColumn[] acols = acolList.toArray( new ArrayColumn[ 0 ] ); + if ( acols.length == 0 ) { + throw new ExecutionException( "No suitable columns " + + "in template array table" ); + } + + /* Create and return a table ready to read the data. */ + return new ArrayDataTable( inTable, acols, alocCompiler, atableLoader, + irow0 ); + } + + /** + * Takes a sequential result table and returns a random-access + * version of the same thing by reading the rows and caching them. + * This also updates the metadata with array length information + * not initially known by the input table. + * + * @param seqTable array-joined table + */ + private static StarTable cacheArrayTable( ArrayDataTable seqTable ) + throws IOException { + + /* Read sequential row data into a cache. */ + RowStore rowStore = StoragePolicy.getDefaultPolicy().makeRowStore(); + rowStore.acceptMetadata( seqTable ); + try ( RowSequence rseq = seqTable.getRowSequence() ) { + while ( rseq.next() ) { + rowStore.acceptRow( rseq.getRow() ); + } + } + rowStore.endRows(); + + /* Store the column metadata and update it with information + * gathered by the ArrayDataTable during row iteration. */ + int nc = seqTable.getColumnCount(); + final ColumnInfo[] cinfos = new ColumnInfo[ nc ]; + for ( int ic = 0; ic < nc; ic++ ) { + cinfos[ ic ] = seqTable.getEnhancedColumnInfo( ic ); + } + + /* Return a table based on the cached table but with enhanced + * column metadata. */ + return new WrapperStarTable( rowStore.getStarTable() ) { + @Override + public ColumnInfo getColumnInfo( int ic ) { + return cinfos[ ic ]; + } + }; + } + + /** + * Reads through an input table to find the first non-blank array + * table associated with one of its rows. + * + * @param inTable input table + * @param alocCompiler produces location of array table + * for input table row + * @param atableLoader loads an array table given a location + * @param rowIndex 1-element array; on exit this will be updated + * to contain the row index in which the first + * table was found + * @return first array table associated with input table + */ + private static StarTable readFirstArrayTable( + StarTable inTable, + Function alocCompiler, + IOFunction atableLoader, + int[] rowIndex ) + throws IOException, TaskException { + try ( SequentialJELRowReader inRdr = + new SequentialJELRowReader( inTable ) ) { + Library lib = JELUtils.getLibrary( inRdr ); + CompiledExpression alocCompex = alocCompiler.apply( lib ); + while ( inRdr.next() ) { + String aloc = evaluateLocation( alocCompex, inRdr ); + if ( aloc != null ) { + StarTable atable = atableLoader.apply( aloc ); + if ( atable != null ) { + return atable; + } + } + } + return null; + } + } + + /** + * Returns the location of an array table associated with the current + * state of a rowReader. + * + * @param locCompex compiled expression yielding table location + * @param rowReader table context within which to evaluate locCompex + * @return array table location + */ + private static String evaluateLocation( CompiledExpression locCompex, + JELRowReader rowReader ) + throws IOException { + try { + return (String) rowReader.evaluate( locCompex ); + } + + /* The message here is not very informative, but I don't really + * expect errors here. If they do show up in practice, think + * about improving the report. */ + catch ( Throwable e ) { + throw new IOException( "Error evaluating table location", e ); + } + } + + /** + * Reads the data from a table as a set of arrays, one containing + * all the data from a table column. + * + * @param acols defines the columns to be read + * @param atable array table to read + * @param irow index of input table for this array table + * @return array of typed arrays, one for each element of acols + */ + private static Object[] readArrayData( ArrayColumn[] acols, + StarTable atable, long irow ) + throws IOException { + int na = acols.length; + + /* It makes life much easier if we know the number of rows. + * Since these tables are not expected to be all that large, + * and we're going to have to read all the data anyway, + * cache the table if that's the only way to find out. */ + int nrow = Tables.checkedLongToInt( atable.getRowCount() ); + if ( nrow < 0 ) { + atable = Tables.randomTable( atable ); + nrow = Tables.checkedLongToInt( atable.getRowCount() ); + } + assert nrow >= 0; + + /* Check the columns look like we are expecting and + * prepare array storage. */ + Object[] adata = new Object[ na ]; + for ( int ia = 0; ia < na; ia++ ) { + ArrayColumn acol = acols[ ia ]; + ColumnInfo gotInfo = atable.getColumnInfo( acol.icol_ ); + ColumnInfo templateInfo = acol.scalarInfo_; + if ( ! gotInfo.getContentClass() + .equals( templateInfo.getContentClass() ) || + ! gotInfo.getName().equals( templateInfo.getName() ) ) { + throw new IOException( "Table data mismatch at row " + irow + + "; " + gotInfo + " does not match " + + templateInfo ); + } + adata[ ia ] = acol.createArray( nrow ); + } + + /* Write all the values into the arrays, a row at a time. */ + try ( RowSequence rseq = atable.getRowSequence() ) { + for ( int ir = 0; ir < nrow; ir++ ) { + rseq.next(); + Object[] row = rseq.getRow(); + for ( int ia = 0; ia < na; ia++ ) { + ArrayColumn acol = acols[ ia ]; + int ic = acol.icol_; + acol.setValueUnchecked( row[ ic ], ir, adata[ ia ] ); + } + } + } + + /* Return populated arrays. */ + return adata; + } + + /** + * Creates an object for managing array data. + * + * @param sInfo metadata for scalar column to be stored in array + * @param icol column index in input table + * @return typed array storage manager + */ + private static ArrayColumn createArrayColumn( ColumnInfo sInfo, + final int icol ) { + Class clazz = sInfo.getContentClass(); + if ( clazz.equals( Boolean.class ) ) { + return new ArrayColumn( boolean[].class, icol, sInfo ) { + boolean[] createArray( int n ) { + return new boolean[ n ]; + } + void setValue( Object value, int ir, boolean[] array ) { + if ( value instanceof Boolean ) { + array[ ir ] = ((Boolean) value).booleanValue(); + } + } + }; + } + else if ( clazz.equals( Byte.class ) ) { + return new ArrayColumn( byte[].class, icol, sInfo ) { + byte[] createArray( int n ) { + return new byte[ n ]; + } + void setValue( Object value, int ir, byte[] array ) { + array[ ir ] = ((Number) value).byteValue(); + } + }; + } + else if ( clazz.equals( Short.class ) ) { + return new ArrayColumn( short[].class, icol, sInfo ) { + short[] createArray( int n ) { + return new short[ n ]; + } + void setValue( Object value, int ir, short[] array ) { + if ( value instanceof Number ) { + array[ ir ] = ((Number) value).shortValue(); + } + } + }; + } + else if ( clazz.equals( Integer.class ) ) { + return new ArrayColumn( int[].class, icol, sInfo ) { + int[] createArray( int n ) { + return new int[ n ]; + } + void setValue( Object value, int ir, int[] array ) { + if ( value instanceof Number ) { + array[ ir ] = ((Number) value).intValue(); + } + } + }; + } + else if ( clazz.equals( Long.class ) ) { + return new ArrayColumn( long[].class, icol, sInfo ) { + long[] createArray( int n ) { + return new long[ n ]; + } + void setValue( Object value, int ir, long[] array ) { + if ( value instanceof Number ) { + array[ ir ] = ((Number) value).longValue(); + } + } + }; + } + else if ( clazz.equals( Float.class ) ) { + return new ArrayColumn( float[].class, icol, sInfo ) { + float[] createArray( int n ) { + return new float[ n ]; + } + void setValue( Object value, int ir, float[] array ) { + array[ ir ] = value instanceof Number + ? ((Number) value).floatValue() + : Float.NaN; + } + }; + } + else if ( clazz.equals( Double.class ) ) { + return new ArrayColumn( double[].class, icol, sInfo ) { + double[] createArray( int n ) { + return new double[ n ]; + } + void setValue( Object value, int ir, double[] array ) { + array[ ir ] = value instanceof Number + ? ((Number) value).doubleValue() + : Double.NaN; + } + }; + } + else if ( clazz.equals( String.class ) ) { + return new ArrayColumn( String[].class, icol, sInfo ) { + String[] createArray( int n ) { + return new String[ n ]; + } + void setValue( Object value, int ir, String[] array ) { + if ( value instanceof String ) { + array[ ir ] = (String) value; + } + } + }; + } + else { + return null; + } + } + + /** + * Class for managing typed array data. + */ + private static abstract class ArrayColumn { + + final Class aClazz_; + final int icol_; + final ColumnInfo scalarInfo_; + final ColumnInfo arrayInfo_; + + /** + * Constructor. + * + * @param aClazz class for array storage + * @param icol index of column in table containing data + * @param scalarInfo metadata for column in table containing data + */ + ArrayColumn( Class aClazz, int icol, ColumnInfo scalarInfo ) { + aClazz_ = aClazz; + icol_ = icol; + scalarInfo_ = scalarInfo; + arrayInfo_ = new ColumnInfo( scalarInfo ); + arrayInfo_.setContentClass( aClazz ); + } + + /** + * Creates an array for storing typed data. + * + * @param n row count + * @return new array + */ + abstract A createArray( int n ); + + /** + * Stores a single element in an storage array. + * + * @param value value to store, expected of suitable type + * @param ix element index + * @param array storage array + */ + abstract void setValue( Object value, int ix, A array ); + + /** + * Convenience untypesafe method for storing data in an array. + * The generics on this class are really to ensure that they + * get implemented without mistakes, not for typechecking + * at usage time which would in any case be a major pain to do. + * + * @param value value to store, expected of suitable type + * @param ix element index + * @param array storage array, assumed of type <A> + */ + final void setValueUnchecked( Object value, int ir, Object array ) { + setValue( value, ir, aClazz_.cast( array ) ); + } + } + + /** + * StarTable implementation for use with this class. + * It is only capable of sequential access. + */ + private static class ArrayDataTable extends AbstractStarTable { + + final StarTable base_; + final ArrayColumn[] arrayCols_; + final Function alocCompiler_; + final IOFunction atableLoader_; + final int irow0_; + final int ncol_; + final static int UNKNOWN_DIM = -1; + final static int VARIABLE_DIM = -2; + int[] arrayDims_; + + /** + * Constructor. + * + * @param base input table + * @param arrayCols array of objects describing the + * columns coming from external array tables + * @param alocCompiler produces location of array table + * for input table row + * @param atableLoader loads an array table given a location + * @param irow0 index of the first row known to contain a + * usable array table; array values can be + * assumed missing for any earlier rows + */ + ArrayDataTable( StarTable base, ArrayColumn[] arrayCols, + Function alocCompiler, + IOFunction atableLoader, + int irow0 ) { + base_ = base; + arrayCols_ = arrayCols; + alocCompiler_ = alocCompiler; + atableLoader_ = atableLoader; + irow0_ = irow0; + ncol_ = arrayCols.length; + } + + public boolean isRandom() { + return false; + } + + public int getColumnCount() { + return ncol_; + } + + public ColumnInfo getColumnInfo( int icol ) { + return arrayCols_[ icol ].arrayInfo_; + } + + public long getRowCount() { + return base_.getRowCount(); + } + + public RowSequence getRowSequence() throws IOException { + SequentialJELRowReader inRdr = new SequentialJELRowReader( base_ ); + Library lib = JELUtils.getLibrary( inRdr ); + final CompiledExpression alocCompex = alocCompiler_.apply( lib ); + final int[] adims = new int[ ncol_ ]; + final Object[] emptyRow = new Object[ ncol_ ]; + Arrays.fill( adims, UNKNOWN_DIM ); + return new RowSequence() { + Object[] adata_; + int irow_ = -1; + public boolean next() throws IOException { + adata_ = null; + if ( inRdr.next() ) { + irow_++; + return true; + } + else { + return false; + } + } + public Object[] getRow() throws IOException { + return getArrayData(); + } + public Object getCell( int icol ) throws IOException { + return getArrayData()[ icol ]; + } + public void close() throws IOException { + adata_ = null; + inRdr.close(); + arrayDims_ = adims.clone(); + } + + /** + * Returns an array of arrays giving the array-valued cells + * at the current row for each of the array columns + * managed by this sequence. + * The data is acquired lazily. + * + * @return ncol-element array of arrays, + * or null if no array data this row + */ + Object[] getArrayData() throws IOException { + if ( adata_ == null ) { + final Object[] adata; + String aloc = evaluateLocation( alocCompex, inRdr ); + if ( aloc != null && irow_ >= irow0_ - 1 ) { + try ( StarTable aTable = + atableLoader_.apply( aloc ) ) { + adata = aTable == null + ? emptyRow + : readArrayData( arrayCols_, aTable, + irow_ ); + } + if ( adata != emptyRow ) { + for ( int ic = 0; ic < ncol_; ic++ ) { + Object array = adata[ ic ]; + if ( array != null ) { + int n = Array.getLength( array ); + if ( n > 0 ) { + if ( adims[ ic ] == UNKNOWN_DIM ) { + adims[ ic ] = n; + } + else if ( adims[ ic ] > 0 && + n != adims[ ic ] ) { + adims[ ic ] = VARIABLE_DIM; + } + assert adims[ ic ] == VARIABLE_DIM + || adims[ ic ] == n; + } + } + } + } + } + else { + adata = null; + } + adata_ = adata; + } + return adata_; + } + }; + } + + /** + * Special method to return the most informative metadata available + * for a given column. This is like getColumnInfo, but + * if a RowSequence has been acquired and completed, + * the dimensions of array columns may be filled in + * (since we've seen the actual lengths of all the array data). + * If called before any row iteration, + * it's just the same as getColumnInfo. + * + * @param icol column index + * @return best available metadata for column + */ + ColumnInfo getEnhancedColumnInfo( int icol ) { + ColumnInfo info = new ColumnInfo( arrayCols_[ icol ].arrayInfo_ ); + int adim = arrayDims_ == null + ? UNKNOWN_DIM + : arrayDims_[ icol ]; + if ( adim > 0 ) { + info.setShape( new int[] { adim } ); + } + return info; + } + + // boilerplate + + public String getName() { + return base_.getName(); + } + public List getParameters() { + return base_.getParameters(); + } + public DescribedValue getParameterByName( String parname ) { + return base_.getParameterByName( parname ); + } + public List getColumnAuxDataInfos() { + return base_.getColumnAuxDataInfos(); + } + public RowSplittable getRowSplittable() throws IOException { + return Tables.getDefaultRowSplittable( this ); + } + } + + /** + * Custom TableInputParameter subclass. + * This just returns a string value (that can eventually get interpreted + * as a table location), but it's worth subclassing + * AbstractInputTableParameter to make use of the associated parameters + * giving table input format etc. + */ + private static class ExpressionInputTableParameter + extends AbstractInputTableParameter { + + /** + * Constructor. + * + * @param name parameter name + */ + ExpressionInputTableParameter( String name ) { + super( name, String.class ); + } + public String stringToObject( Environment env, String sval ) { + return sval; + } + } +} diff -Nru starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/votlint/UcdChecker.java starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/votlint/UcdChecker.java --- starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/votlint/UcdChecker.java 2022-06-10 13:14:05.000000000 +0000 +++ starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/votlint/UcdChecker.java 2022-07-08 09:59:19.000000000 +0000 @@ -1,5 +1,7 @@ package uk.ac.starlink.ttools.votlint; +import uk.ac.starlink.vo.UcdStatus; + /** * Attribute checker for checking UCD syntax. * diff -Nru starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/votlint/UcdStatus.java starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/votlint/UcdStatus.java --- starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/votlint/UcdStatus.java 2022-06-10 13:14:05.000000000 +0000 +++ starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/votlint/UcdStatus.java 1970-01-01 00:00:00.000000000 +0000 @@ -1,272 +0,0 @@ -package uk.ac.starlink.ttools.votlint; - -import ari.ucidy.UCD; -import ari.ucidy.UCDParser; -import ari.ucidy.UCDWord; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.Set; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.regex.Pattern; - -/** - * Categorises UCD validity. - * The {@link #getStatus} method tests a UCD string to determine whether - * it conforms to standards, and returns an object that reports this. - * UCD1+ and UCD1 are recognised. - * - *

Gregory Mantelet's Ucidy library is used for UCD1+ parsing. - * - * @author Mark Taylor - * @since 9 Jul 2021 - */ -public class UcdStatus { - - private final Code code_; - private final String message_; - - private static final Map statusMap_ = createCache( 200 ); - private static final UCDParser ucdParser_ = createParser(); - private static final Collection ucd1s_ = - Collections.unmodifiableSet( readUcd1s() ); - private static final Collection ucd1Or1plusses_ = - Collections.unmodifiableSet( new HashSet<>( Arrays.asList( new String[]{ - "PHYS", "POS", "SPECT", "STAT", "TIME", - } ) ) ); - private static final Pattern VOX_REGEX = - Pattern.compile( "VOX:[A-Za-z_]+" ); - - /** - * Constructor. - * - * @param code status code - * @param message human-readable message supplying additional information - */ - protected UcdStatus( Code code, String message ) { - code_ = code; - message_ = message; - } - - /** - * Returns a status element indicating conformance. - * - * @return code - */ - public Code getCode() { - return code_; - } - - /** - * Returns a human-readable message supplying additional information. - * The returned text does not in general report the original UCD string. - * - * @return message, or null if nothing to say (OK status) - */ - public String getMessage() { - return message_; - } - - /** - * Returns the status for a given UCD string. - * - * @param ucd UCD text - * @return status, or null for blank input - */ - public static UcdStatus getStatus( String ucd ) { - return statusMap_.computeIfAbsent( ucd, UcdStatus::createStatus ); - } - - /** - * Returns the UCD parser used by this class. - * - * @return parser - */ - public static UCDParser getParser() { - return ucdParser_; - } - - /** - * Returns a UCDParser instance for use by this class. - * - * @return parser - */ - private static UCDParser createParser() { - - /* At time of writing there is an issue with the list of deprecated - * words that means a message is issued. We will ignore this. - * Future ucidy updates (based perhaps on UCD1+ errata or versions - * later than 1.4) may mean this is no longer necessary. */ - Logger.getLogger( "ari.ucidy" ).setLevel( Level.OFF ); - - /* Currently use the default instance, but could construct one - * based on a customised word list. */ - return UCDParser.defaultParser; - } - - /** - * Does the work of parsing the UCD. - * - * @param ucd UCD text - * @return new status, or null for blank input - */ - private static UcdStatus createStatus( String ucd ) { - if ( ucd == null || ucd.trim().length() == 0 ) { - return null; - } - if ( ucd1s_.contains( ucd.toUpperCase() ) && - ! ucd1Or1plusses_.contains( ucd.toUpperCase() ) ) { - return new UcdStatus( Code.UCD1, "UCD1, not UCD1+" ); - } - if ( VOX_REGEX.matcher( ucd ).matches() ) { - return new UcdStatus( Code.VOX, "SIAv1-style VOX namespace" ); - } - UCD pucd = ucdParser_.parse( ucd ); - StringBuffer sbuf = new StringBuffer(); - for ( Iterator it = pucd.getErrors(); it.hasNext(); ) { - String line = it.next().replaceFirst( " *!$", "" ); - if ( sbuf.length() > 0 && line.length() > 0 ) { - sbuf.append( "; " ); - } - sbuf.append( line ); - } - String message = sbuf.length() > 0 ? sbuf.toString() : null; - boolean hasNamespace = false; - boolean isValid = true; - boolean isRecognised = true; - boolean isRecommended = true; - boolean isDeprecated = false; - for ( UCDWord word : pucd ) { - if ( word != null ) { - hasNamespace = hasNamespace || word.namespace != null; - isValid = isValid && word.valid; - isRecognised = isRecognised && word.recognised; - isRecommended = isRecommended && word.recommended; - isDeprecated = isDeprecated || word.isDeprecated(); - } - } - if ( ! isValid ) { - return new UcdStatus( Code.BAD_SYNTAX, message ); - } - if ( hasNamespace ) { - return new UcdStatus( Code.NAMESPACE, message ); - } - if ( isDeprecated ) { - return new UcdStatus( Code.DEPRECATED, message ); - } - if ( ! isRecognised ) { - return new UcdStatus( Code.UNKNOWN_WORD, message ); - } - if ( ! isRecommended ) { - return new UcdStatus( Code.DEPRECATED, message ); - } - return new UcdStatus( pucd.isFullyValid() ? Code.OK : Code.BAD_SEQUENCE, - message ); - } - - /** - * Returns a collection of known UCD1 strings. - * - * @return UCD1s - */ - private static Set readUcd1s() { - Set set = new LinkedHashSet<>(); - for ( Iterator it = - uk.ac.starlink.table.UCD.getUCDs(); it.hasNext(); ) { - set.add( it.next().getID() ); - } - return set; - } - - /** - * Returns an LRU cache suitable for storing UcdStatus values. - * - * @param limit maximum cache size - * @return new thread-safe map - */ - private static Map createCache( final int limit ) { - return Collections - .synchronizedMap( new LinkedHashMap() { - @Override - public boolean removeEldestEntry( Map.Entry - entry ) { - return size() > limit; - } - } ); - } - - /** - * Characterises UCD standards conformance. - */ - public enum Code { - - /** Conforms to UCD1+ standard. */ - OK( ' ' ), - - /** Conforms to UCD1 standard. */ - UCD1( ' ' ), - - /** Is in VOX: namespace introduced by SIAv1. */ - VOX( ' ' ), - - /** Not a UCD1 and cannot be parsed according to UCD1+. */ - BAD_SYNTAX( 'E' ), - - /** UCD words violate UCD1+ sequence rules. */ - BAD_SEQUENCE( 'E' ), - - /** UCD1+ syntax but contains non-UCD1+ word. */ - UNKNOWN_WORD( 'E' ), - - /** Contains apparently namespaced UCD words. */ - NAMESPACE( ' ' ), - - /** Contains deprecated UCD1+ words. */ - DEPRECATED( 'W' ); - - private final char stat_; - - /** - * Constructor. - * - * @param stat status code: [E]rror, [W]arning or ' ' (ok) - */ - private Code( char stat ) { - stat_ = stat; - switch ( stat ) { - case ' ': - case 'W': - case 'E': - break; - default: - throw new IllegalArgumentException(); - } - } - - /** - * Indicates whether this status represents a UCD value which - * violates known standards. - * - * @return true for error status - */ - public boolean isError() { - return stat_ == 'E'; - } - - /** - * Indicates whether this status represents a UCD value which - * may deserve attention, but is not actually a standards violation. - * - * @return true for warning status - */ - public boolean isWarning() { - return stat_ == 'W'; - } - } -} diff -Nru starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/votlint/UnitChecker.java starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/votlint/UnitChecker.java --- starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/votlint/UnitChecker.java 2022-06-10 13:14:05.000000000 +0000 +++ starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/votlint/UnitChecker.java 2022-07-08 09:59:19.000000000 +0000 @@ -1,5 +1,7 @@ package uk.ac.starlink.ttools.votlint; +import uk.ac.starlink.vo.UnitStatus; + /** * Attribute checker that checks unit strings against the VOUnits standard. * diff -Nru starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/votlint/UnitStatus.java starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/votlint/UnitStatus.java --- starjava-ttools-3.4.5/src/main/uk/ac/starlink/ttools/votlint/UnitStatus.java 2022-06-10 13:14:05.000000000 +0000 +++ starjava-ttools-3.4.6/src/main/uk/ac/starlink/ttools/votlint/UnitStatus.java 1970-01-01 00:00:00.000000000 +0000 @@ -1,250 +0,0 @@ -package uk.ac.starlink.ttools.votlint; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import uk.me.nxg.unity.OneUnit; -import uk.me.nxg.unity.Syntax; -import uk.me.nxg.unity.UnitDefinition; -import uk.me.nxg.unity.UnitExpr; -import uk.me.nxg.unity.UnitParser; -import uk.me.nxg.unity.UnitParserException; -import uk.me.nxg.unity.UnitRepresentation; - -/** - * Categorises VOUnit validity. - * The {@link #getStatus} method tests a unit string to determine whether - * it conforms to the VOUnits standard. - * - *

Norman Gray's Unity library is used for VOUnits parsing. - * - * @author Mark Taylor - * @since 9 Jul 2021 - * @see VOUnits - */ -public class UnitStatus { - - private final Code code_; - private final String message_; - private static final Map statusMap_ = createCache( 200 ); - - /** - * Constructor. - * - * @param code status code - * @param message human-readable message supplying additional information - */ - protected UnitStatus( Code code, String message ) { - code_ = code; - message_ = message; - } - - /** - * Returns a status element indicating conformance. - * - * @return code - */ - public Code getCode() { - return code_; - } - - /** - * Returns a human-readable message supplying additional information. - * The returned text does not in general report the original unit string. - * - * @return message, or null if nothing to say - */ - public String getMessage() { - return message_; - } - - /** - * Returns the status for a given unit string. - * - * @param unit unit text - * @return status, or null for blank input - */ - public static UnitStatus getStatus( String unit ) { - return statusMap_.computeIfAbsent( unit, UnitStatus::createStatus ); - } - - /** - * Does the work of parsing a Unit. - * - * @param unit unit text - * @return new status, or null for blank input - */ - private static UnitStatus createStatus( String unit ) { - Syntax syntax = Syntax.VOUNITS; - if ( unit == null || unit.trim().length() == 0 ) { - return null; - } - String cunit = unit.replaceAll( "\\s", "" ); - boolean hasWhitespace = ! cunit.equals( unit ); - UnitExpr punit; - try { - UnitParser parser = new UnitParser( syntax ); - parser.setGuessing( true ); - punit = parser.parse( cunit ); - } - catch ( UnitParserException e ) { - return new UnitStatus( Code.BAD_SYNTAX, e.getMessage() ); - } - catch ( Throwable e ) { - return new UnitStatus( Code.PARSE_ERROR, e.toString() ); - } - if ( hasWhitespace ) { - return new UnitStatus( Code.WHITESPACE, - "Whitespace illegal at VOUnits 1.0" ); - } - Map unknown = new LinkedHashMap<>(); - List deprecated = new ArrayList<>(); - for ( OneUnit word : punit ) { - if ( ! word.isRecognisedUnit( syntax ) || word.wasGuessed() ) { - String utxt = word.getOriginalUnitString(); - UnitDefinition udef = word.getBaseUnitDefinition(); - UnitRepresentation urep = udef == null - ? null - : udef.getRepresentation( syntax ); - String guess = urep == null ? null : urep.toString(); - unknown.put( utxt, guess ); - } - if ( ! word.isRecommendedUnit( syntax ) ) { - deprecated.add( word.getBaseUnitName() ); - } - } - if ( unknown.size() > 0 ) { - List items = new ArrayList<>(); - boolean allGuessed = true; - for ( Map.Entry unkEntry : unknown.entrySet() ) { - String utxt = unkEntry.getKey(); - String guess = unkEntry.getValue(); - StringBuffer sbuf = new StringBuffer() - .append( '"' ) - .append( utxt ) - .append( '"' ); - if ( guess != null ) { - sbuf.append( " (" ) - .append( "-> \"" ) - .append( guess ) - .append( "\"?)" ); - } - else { - allGuessed = false; - } - items.add( sbuf.toString() ); - } - String txt = ( unknown.size() == 1 ? "Unknown unit " - : "Unknown units " ) - + String.join( ", ", items ); - return new UnitStatus( allGuessed ? Code.GUESSED_UNIT - : Code.UNKNOWN_UNIT, - txt ); - } - if ( ! punit.allUsageConstraintsSatisfied( syntax ) ) { - return new UnitStatus( Code.USAGE, "Usage constraints violated" ); - } - if ( deprecated.size() > 0 ) { - String txt = deprecated.size() == 1 - ? "Deprecated unit \"" + deprecated.get( 0 ) + "\"" - : "Deprecated units " + deprecated; - return new UnitStatus( Code.DEPRECATED, txt ); - } - assert punit.isFullyConformant( syntax ); - return new UnitStatus( Code.OK, null ); - } - - /** - * Returns an LRU cache suitable for storing UcdStatus values. - * - * @param limit maximum cache size - * @return new thread-safe map - */ - private static Map createCache( final int limit ) { - return Collections - .synchronizedMap( new LinkedHashMap() { - @Override - public boolean removeEldestEntry( Map.Entry - entry ) { - return size() > limit; - } - } ); - } - - /** - * Characterises VOUnits standard conformance. - */ - public enum Code { - - /** Conforms to VOUnits standard. */ - OK( ' ' ), - - /** Contains some units deprecated in VOUnits standard. */ - // In most cases this means deprecated by the IAU. - // Don't even class this as a warning, since it contains some - // items like Angstrom and erg that are common and reasonable. - DEPRECATED( ' ' ), - - /** Parsed as VOUnit but contains unknown base units. */ - UNKNOWN_UNIT( 'W' ), - - /** Parsed as VOUnit but contains unknown though guessable units. */ - GUESSED_UNIT( 'W' ), - - /** Cannot be parsed as VOUnit. */ - BAD_SYNTAX( 'E' ), - - /** Cannot be parsed as VOUnit (shouldn't happen). */ - PARSE_ERROR( 'E' ), - - /** Violates VOUnit usage constraints. */ - USAGE( 'E' ), - - /** Legal VOUnit except that it contains illegal whitespace. */ - WHITESPACE( 'E' ); - - private final char stat_; - - /** - * Constructor. - * - * @param stat status code: [E]rror, [W]arning or ' ' (ok) - */ - private Code( char stat ) { - stat_ = stat; - switch ( stat ) { - case ' ': - case 'W': - case 'E': - break; - default: - throw new IllegalArgumentException(); - } - } - - /** - * Indicates whether this code represents an invalid VOUnit - * specification. - * - * @return true for error status - */ - public boolean isError() { - return stat_ == 'E'; - } - - /** - * Indicates whether this status represents a UCD value which - * may deserve attention, but is not actually a standards violation. - * Note this includes use of unknown and non-standard, - * as well as deprecated, units that are still syntactically - * permissible. - * - * @return true for warning status - */ - public boolean isWarning() { - return stat_ == 'W'; - } - } -} diff -Nru starjava-ttools-3.4.5/src/testcases/uk/ac/starlink/ttools/func/FuncTest.java starjava-ttools-3.4.6/src/testcases/uk/ac/starlink/ttools/func/FuncTest.java --- starjava-ttools-3.4.5/src/testcases/uk/ac/starlink/ttools/func/FuncTest.java 2022-06-10 13:14:05.000000000 +0000 +++ starjava-ttools-3.4.6/src/testcases/uk/ac/starlink/ttools/func/FuncTest.java 2022-07-08 09:59:19.000000000 +0000 @@ -220,12 +220,19 @@ assertArrayEquals( new double[] { 3, 4, Double.NaN }, Arrays.add( new float[] { 1f, 2f, Float.NaN, }, 2 ) ); + assertArrayEquals( new double[] { 3, 4, Double.NaN }, + Arrays.add( 2, + new float[] { 1f, 2f, Float.NaN, } ) ); assertNull( Arrays.add( "no array", new int[] { 1, 3 } ) ); assertNull( Arrays.add( new int[] { 1, 3 }, "no array" ) ); + assertNull( Arrays.add( new double[ 10 ], new double[ 11 ] ) ); assertArrayEquals( new double[] { 1.0, 1.75, 2.5 }, Arrays.subtract( new int[] { 1, 2, 3 }, new float[] { 0, 0.25f, 0.5f } ) ); + assertArrayEquals( new double[] { 7, 6, 5 }, + Arrays.subtract( new float[] { 10, 9, 8 }, + (short) 3 ) ); assertArrayEquals( new double[] { 0.5, 3.0, Double.NaN }, Arrays.multiply( new int[] { 1, 2, 3, }, @@ -234,15 +241,25 @@ assertArrayEquals( new double[] { 2, 4, Double.NaN }, Arrays.multiply( new float[] { 1f, 2f, Float.NaN, }, 2 ) ); + assertArrayEquals( new double[] { 2, 4, Double.NaN }, + Arrays.multiply( 2, + new float[] { 1f, 2f, Float.NaN} )); assertNull( Arrays.multiply( "no array", new int[] { 1, 3 } ) ); assertNull( Arrays.multiply( new int[] { 1, 3 }, "no array" ) ); + assertEquals( 26, Arrays.dotProduct( new double[] { 3, 4, 5 }, + new short[] { 1, 2, 3 } ) ); + assertTrue( Double.isNaN( Arrays.dotProduct( new float[ 2 ], + new float[ 3 ] ) ) ); + assertArrayEquals( new double[] { 3, 4, 3 }, Arrays .condition( new boolean[] { true, false, true }, 3, 4 ) ); assertArrayEquals( new double[] { 1, 0.5, 4 }, Arrays.reciprocal( new float[] { 1, 2, 0.25f, } ) ); + assertArrayEquals( new double[] { 1, 0.5, 4 }, + Arrays.divide( 1, new float[] { 1, 2, 0.25f, } ) ); assertArrayEquals( new double[] { 0, 3, 0.5 }, Arrays.divide( new short[] { 0, 9, 4 }, new double[] { 1, 3, 8 } ) ); diff -Nru starjava-ttools-3.4.5/src/testcases/uk/ac/starlink/ttools/jel/JELTest.java starjava-ttools-3.4.6/src/testcases/uk/ac/starlink/ttools/jel/JELTest.java --- starjava-ttools-3.4.5/src/testcases/uk/ac/starlink/ttools/jel/JELTest.java 2022-06-10 13:14:05.000000000 +0000 +++ starjava-ttools-3.4.6/src/testcases/uk/ac/starlink/ttools/jel/JELTest.java 2022-07-08 09:59:19.000000000 +0000 @@ -224,6 +224,59 @@ .evaluate_boolean( null ) ); } + public void testValueFunctions() throws Throwable { + StarTable t1 = new QuickTable( 3, new ColumnData[] { + col( "Source", new long[] { 4656637047367315584L, + 6098162212326263680L, + 2367940784546457472L, } ), + col( "Target", new String[] { "g0435359-671024", + "g1431504-470630", + null } ), + col( "[Fe/H]", new float[] { -0.8388f, 4.541f, Float.NaN, } ), + col( "b_[Fe/H]", new float[] { -0.8694f, -0.4985f, Float.NaN, } ), + col( "B_[Fe/H]", new float[] { -0.8061f, -0.4131f, Float.NaN, } ), + } ); + assertEquals( "g1431504-470630", + evaluate( t1, "valueString(\"Target\")", 1 ) ); + assertEquals( "g1431504-470630", + evaluate( t1, "valueObject(\"Target\")", 1 ) ); + assertTrue( Tables + .isBlank( evaluate( t1, "valueDouble(\"Target\")", 1 ) ) ); + assertNull( evaluate( t1, "valueInt(\"Target\")", 1 ) ); + assertNull( evaluate( t1, "valueLong(\"Target\")", 1 ) ); + + assertNull( evaluate( t1, "valueString(\"target\")", 1 ) ); + assertNull( evaluate( t1, "valueObject(\"target\")", 1 ) ); + assertNull( evaluate( t1, "valueInt(\"target\")", 1 ) ); + assertNull( evaluate( t1, "valueLong(\"target\")", 1 ) ); + assertTrue( Tables + .isBlank( evaluate( t1, "valueDouble(\"target\")", 1 ) ) ); + + assertEquals( Float.valueOf( -0.8388f ), + evaluate( t1, "(float)valueDouble(\"[Fe/H]\")", 0 ) ); + assertEquals( Integer.valueOf( 4 ), + evaluate( t1, "valueInt(\"[Fe/H]\")", 1 ) ); + assertEquals( Long.valueOf( 4 ), + evaluate( t1, "valueLong(\"[Fe/H]\")", 1 ) ); + assertNull( evaluate( t1, "valueInt(\"[Fe/H]\")", 2 ) ); + assertNull( evaluate( t1, "valueLong(\"[Fe/H]\")", 2 ) ); + assertTrue( Tables + .isBlank( evaluate( t1, "valueDouble(\"[Fe/H]\")", 2 ) ) ); + + assertEquals( Float.valueOf( -0.8694f ), + evaluate( t1, "(float)valueDouble(\"b_[Fe/H]\")", 0 ) ); + assertEquals( Float.valueOf( -0.8061f ), + evaluate( t1, "(float)valueDouble(\"B_[Fe/H]\")", 0 ) ); + } + + private Object evaluate( StarTable table, String expr, long irow ) + throws Throwable { + RandomJELRowReader rdr = RandomJELRowReader.createAccessReader( table ); + Library lib = JELUtils.getLibrary( rdr ); + CompiledExpression compex = JELUtils.compile( lib, table, expr ); + return rdr.evaluateAtRow( compex, irow ); + } + public static class FuncLib { public static int triplePrim( int a ) { return 3 * a; diff -Nru starjava-ttools-3.4.5/src/testcases/uk/ac/starlink/ttools/task/ArrayJoinTest.java starjava-ttools-3.4.6/src/testcases/uk/ac/starlink/ttools/task/ArrayJoinTest.java --- starjava-ttools-3.4.5/src/testcases/uk/ac/starlink/ttools/task/ArrayJoinTest.java 1970-01-01 00:00:00.000000000 +0000 +++ starjava-ttools-3.4.6/src/testcases/uk/ac/starlink/ttools/task/ArrayJoinTest.java 2022-07-08 09:59:19.000000000 +0000 @@ -0,0 +1,59 @@ +package uk.ac.starlink.ttools.task; + +import java.lang.reflect.Array; +import java.util.logging.Level; +import java.util.logging.Logger; +import uk.ac.starlink.table.StarTable; +import uk.ac.starlink.table.Tables; +import uk.ac.starlink.ttools.TableTestCase; + +public class ArrayJoinTest extends TableTestCase { + + private final Logger logger_; + + public ArrayJoinTest() { + logger_ = Logger.getLogger( "uk.ac.starlink" ); + logger_.setLevel( Level.WARNING ); + } + + public void testArrayJoin() throws Exception { + exerciseArrayJoin( true ); + exerciseArrayJoin( false ); + } + + private void exerciseArrayJoin( boolean isCached ) throws Exception { + MapEnvironment env = new MapEnvironment(); + env.setValue( "in", ":loop:6" ); + env.setValue( "atable", "$0%3==1?\"x\":(\":loop:\"+$0)" ); + env.setValue( "acmd", "addcol x (int)$0; addcol y 1.0*x*x" ); + env.setValue( "cache", Boolean.valueOf( isCached ) ); + env.setValue( "fixcols", "dups" ); + env.setValue( "suffixarray", "_array" ); + new ArrayJoin().createExecutable( env ).execute(); + StarTable tout = env.getOutputTable( "omode" ); + assertTrue( tout.isRandom() == isCached ); + Tables.checkTable( tout ); + tout = Tables.randomTable( tout ); + assertEquals( 6, tout.getRowCount() ); + assertArrayEquals( new String[] { "i", "i_array", "x", "y" }, + getColNames( tout ) ); + + Object[] row3 = tout.getRow( 3 ); + assertArrayEquals( + new Object[] { Integer.valueOf( 3 ), null, null, null }, + row3 ); + + Object[] row2 = tout.getRow( 2 ); + assertEquals( Integer.valueOf( 2 ), row2[ 0 ] ); + assertArrayEquals( new int[] { 0, 1, 2, }, row2[ 1 ] ); + assertArrayEquals( new int[] { 1, 2, 3, }, row2[ 2 ] ); + assertArrayEquals( new double[] { 1., 4., 9., }, row2[ 3 ] ); + + for ( int ir = 4; ir < 6; ir++ ) { + for ( int ic = 1; ic < 4; ic++ ) { + assertEquals( ir + 1, + Array.getLength( tout.getCell( ir, ic ) ) ); + } + } + } +} diff -Nru starjava-ttools-3.4.5/src/testcases/uk/ac/starlink/ttools/task/ParameterTest.java starjava-ttools-3.4.6/src/testcases/uk/ac/starlink/ttools/task/ParameterTest.java --- starjava-ttools-3.4.5/src/testcases/uk/ac/starlink/ttools/task/ParameterTest.java 2022-06-10 13:14:05.000000000 +0000 +++ starjava-ttools-3.4.6/src/testcases/uk/ac/starlink/ttools/task/ParameterTest.java 2022-07-08 09:59:19.000000000 +0000 @@ -30,6 +30,7 @@ "cols", "binsizes", "coords", "serviceurl", "ra", "dec", "sr", "tapurl", "joburl", "adql", "uploadN", "votable", + "atable", "query", "cache", "href", "db", "dbtable", "dbra", "dbdec", diff -Nru starjava-ttools-3.4.5/src/testcases/uk/ac/starlink/ttools/votlint/CodeTest.java starjava-ttools-3.4.6/src/testcases/uk/ac/starlink/ttools/votlint/CodeTest.java --- starjava-ttools-3.4.5/src/testcases/uk/ac/starlink/ttools/votlint/CodeTest.java 2022-06-10 13:14:05.000000000 +0000 +++ starjava-ttools-3.4.6/src/testcases/uk/ac/starlink/ttools/votlint/CodeTest.java 1970-01-01 00:00:00.000000000 +0000 @@ -1,84 +0,0 @@ -package uk.ac.starlink.ttools.votlint; - -import junit.framework.TestCase; - -public class CodeTest extends TestCase { - - public void testUcdStatus() { - for ( String ucd : new String[] { - "pos.eq.ra;meta.main", - "stat.likelihood", - "TIME.PERIOD;STAT.MAX", - "pos.eq.ra;meta.main", - } ) { - UcdStatus status = UcdStatus.getStatus( ucd ); - UcdStatus.Code code = status.getCode(); - assertEquals( UcdStatus.Code.OK, code ); - assertFalse( code.isError() ); - assertFalse( code.isWarning() ); - assertNull( status.getMessage(), status.getMessage() ); - } - for ( String ucd : new String[] { - "pos.earth;pos.bodyrc.lat", - "pos.bodyrc", - "pump.m.up", - "POS_EQ_RA_MAIN;META_MAIN", - } ) { - UcdStatus status = UcdStatus.getStatus( ucd ); - UcdStatus.Code code = status.getCode(); - assertTrue( code.isError() ); - assertFalse( code.isWarning() ); - assertTrue( status.getMessage().length() > 0 ); - } - UcdStatus ucd1Status = UcdStatus.getStatus( "POS_EQ_DEC_MAIN" ); - assertEquals( UcdStatus.Code.UCD1, ucd1Status.getCode() ); - assertTrue( ucd1Status.getMessage().indexOf( "UCD1" ) >= 0 ); - - UcdStatus deprecStatus = UcdStatus.getStatus( "time.expo" ); - UcdStatus.Code deprecCode = deprecStatus.getCode(); - assertEquals( UcdStatus.Code.DEPRECATED, deprecCode ); - assertTrue( deprecStatus.getMessage().indexOf( "DEPRECATED" ) >= 0 ); - assertFalse( deprecCode.isError() ); - assertTrue( deprecCode.isWarning() ); - } - - public void testUnitStatus() { - for ( String unit : new String[] { - "m", - "m**2", - "m**-2/s", - "m.N", - "mN", - } ) { - UnitStatus status = UnitStatus.getStatus( unit ); - UnitStatus.Code code = status.getCode(); - assertEquals( UnitStatus.Code.OK, code ); - assertFalse( code.isError() ); - assertNull( status.getMessage() ); - } - for ( String unit : new String[] { - "m^2", - "m**-2 / s", - } ) { - UnitStatus status = UnitStatus.getStatus( unit ); - UnitStatus.Code code = status.getCode(); - assertTrue( code.isError() ); - assertTrue( status.getMessage().length() > 0 ); - } - for ( String unit : new String[] { - "'electron'.pix**-1", - "sizeOfWales", - } ) { - UnitStatus status = UnitStatus.getStatus( unit ); - UnitStatus.Code code = status.getCode(); - assertFalse( code.isError() ); - assertTrue( code.isWarning() ); - assertTrue( status.getMessage().length() > 0 ); - } - assertEquals( UnitStatus.Code.WHITESPACE, - UnitStatus.getStatus( "m / s" ).getCode() ); - assertEquals( UnitStatus.Code.DEPRECATED, - UnitStatus.getStatus( "Angstrom" ).getCode() ); - } - -}