From 5ff4911ec6820a900ef3a9f1c696371ec072db00 Mon Sep 17 00:00:00 2001 From: Eric Wei Date: Fri, 28 Aug 2026 19:00:58 +0000 Subject: [PATCH 1/2] test(integ-test): make first/last/take tests independent of document order first() and last() select by input-stream position, which is not defined across shards. Tests that asserted a specific document are made deterministic in one of two ways. Tests whose intent is stream-position semantics build the stream with makeresults, so the order is defined by the literal rows. Tests whose intent is index-backed field access (text, deep nested, alias, script) stay on the index and narrow to a single candidate document per output cell, keeping the field-access path under test. No production code is changed. Signed-off-by: Eric Wei --- .../CalciteAliasFieldAggregationIT.java | 60 +++-- .../remote/CalcitePPLAggregationIT.java | 226 +++++++++++------- 2 files changed, 169 insertions(+), 117 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java index 5a23492ca6e..63d2a433c5f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java @@ -109,7 +109,9 @@ public void testMinMaxWithNumericAliasField() throws IOException { public void testFirstWithAliasField() throws IOException { JSONObject actual = executeQuery( - String.format("source=%s | sort @timestamp | stats FIRST(@timestamp)", TEST_ALIAS_BUG)); + String.format( + "source=%s | where value = 100 | sort @timestamp | stats FIRST(@timestamp)", + TEST_ALIAS_BUG)); verifySchema(actual, schema("FIRST(@timestamp)", "timestamp")); verifyDataRows(actual, rows("2024-01-01 10:00:00")); } @@ -118,7 +120,9 @@ public void testFirstWithAliasField() throws IOException { public void testLastWithAliasField() throws IOException { JSONObject actual = executeQuery( - String.format("source=%s | sort @timestamp | stats LAST(@timestamp)", TEST_ALIAS_BUG)); + String.format( + "source=%s | where value = 300 | sort @timestamp | stats LAST(@timestamp)", + TEST_ALIAS_BUG)); verifySchema(actual, schema("LAST(@timestamp)", "timestamp")); verifyDataRows(actual, rows("2024-01-03 10:00:00")); } @@ -128,9 +132,10 @@ public void testTakeWithAliasField() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | sort @timestamp | stats TAKE(@timestamp, 2)", TEST_ALIAS_BUG)); + "source=%s | where value = 100 | sort @timestamp | stats TAKE(@timestamp, 2)", + TEST_ALIAS_BUG)); verifySchema(actual, schema("TAKE(@timestamp, 2)", "array")); - verifyDataRows(actual, rows(List.of("2024-01-01T10:00:00.000Z", "2024-01-02T10:00:00.000Z"))); + verifyDataRows(actual, rows(List.of("2024-01-01 10:00:00"))); } @Test @@ -160,39 +165,42 @@ public void testUnaffectedAggregationsWithAliasFields() throws IOException { @Test public void testAliasTypeWithLastFirstTakeLatestEarliestAggregation() throws IOException { + // take/first/last select by document order, which is not defined across shards, so they are + // asserted against a single narrowed document. latest/earliest are ordered by the time field + // and + // stay deterministic, so they keep asserting over the full dataset. JSONObject actual = executeQuery( String.format( - "source=%s | stats take(original_text, 2), last(original_text)," - + " first(original_text), take(alias_text, 2), last(alias_text)," - + " first(alias_text), take(original_col, 2), last(original_col)," - + " first(original_col), take(alias_col, 2), last(alias_col), first(alias_col)," - + " latest(original_col), earliest(original_col), latest(alias_col)," - + " earliest(alias_col),latest(original_text), earliest(original_text)," - + " latest(alias_text), earliest(alias_text)", + "source=%s | where original_col = 1 | stats take(original_text, 2)," + + " last(original_text), first(original_text), take(alias_text, 2)," + + " last(alias_text), first(alias_text), take(original_col, 2)," + + " last(original_col), first(original_col), take(alias_col, 2)," + + " last(alias_col), first(alias_col)", TEST_INDEX_ALIAS)); verifyDataRows( actual, rows( - List.of("a b c", "d e f"), - "x y z", + List.of("a b c"), "a b c", - List.of("a b c", "d e f"), - "x y z", "a b c", - List.of(1, 2), - 3, - 1, - List.of(1, 2), - 3, + List.of("a b c"), + "a b c", + "a b c", + List.of(1), 1, - 3, 1, - 3, + List.of(1), 1, - "x y z", - "a b c", - "x y z", - "a b c")); + 1)); + + JSONObject ordered = + executeQuery( + String.format( + "source=%s | stats latest(original_col), earliest(original_col)," + + " latest(alias_col), earliest(alias_col), latest(original_text)," + + " earliest(original_text), latest(alias_text), earliest(alias_text)", + TEST_INDEX_ALIAS)); + verifyDataRows(ordered, rows(3, 1, 3, 1, "x y z", "a b c", "x y z", "a b c")); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java index f043d9b3afc..f839491c6e8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java @@ -39,6 +39,24 @@ public class CalcitePPLAggregationIT extends PPLIntegTestCase { private static final String TEST_INDEX_TIME_DATA = "opensearch-sql_test_index_time_data"; + /** + * first() and last() select by position in the input stream. Document order across a multi-shard + * index is not defined, so tests that assert stream position build the stream deterministically + * with makeresults rather than relying on index document order. The rows mirror the bank fixture + * in its indexed order, so the expected values are the same ones the index-backed queries + * produced on a single shard. + */ + private static final String BANK_STREAM = + "makeresults format=csv data='account_number:long,firstname:string,lastname:string," + + "employer:string,gender:string,age:int,balance:long\\n" + + "1,Amber JOHnny,Duke Willmington,Pyrami,M,32,39225\\n" + + "6,Hattie,Bond,Netagy,M,36,5686\\n" + + "13,Nanette,Bates,Quility,F,28,32838\\n" + + "18,Dale,Adams,Boink,M,33,4180\\n" + + "20,Elinor,Ratliff,Scentric,M,36,16418\\n" + + "25,Virginia,Ayala,Filodyne,F,39,40540\\n" + + "32,Dillard,Mcpherson,Quailcom,F,34,48086'"; + @Override public void init() throws Exception { super.init(); @@ -415,16 +433,14 @@ public void testAvgBySpan() throws IOException { @Test public void testFirstAggregation() throws IOException { - JSONObject actual = - executeQuery(String.format("source=%s | stats first(firstname)", TEST_INDEX_BANK)); + JSONObject actual = executeQuery(BANK_STREAM + " | stats first(firstname)"); verifySchema(actual, schema("first(firstname)", "string")); verifyDataRows(actual, rows("Amber JOHnny")); } @Test public void testLastAggregation() throws IOException { - JSONObject actual = - executeQuery(String.format("source=%s | stats last(firstname)", TEST_INDEX_BANK)); + JSONObject actual = executeQuery(BANK_STREAM + " | stats last(firstname)"); verifySchema(actual, schema("last(firstname)", "string")); verifyDataRows(actual, rows("Dillard")); } @@ -432,9 +448,7 @@ public void testLastAggregation() throws IOException { @Test public void testFirstLastByGroup() throws IOException { JSONObject actual = - executeQuery( - String.format( - "source=%s | stats first(firstname), last(lastname) by gender", TEST_INDEX_BANK)); + executeQuery(BANK_STREAM + " | stats first(firstname), last(lastname) by gender"); verifySchema( actual, schema("first(firstname)", "string"), @@ -447,9 +461,8 @@ public void testFirstLastByGroup() throws IOException { public void testFirstLastWithOtherAggregations() throws IOException { JSONObject actual = executeQuery( - String.format( - "source=%s | stats first(firstname), last(firstname), count(), avg(age) by gender", - TEST_INDEX_BANK)); + BANK_STREAM + + " | stats first(firstname), last(firstname), count(), avg(age) by gender"); verifySchema( actual, schema("first(firstname)", "string"), @@ -466,10 +479,7 @@ public void testFirstLastWithOtherAggregations() throws IOException { @Test public void testFirstLastDifferentFields() throws IOException { JSONObject actual = - executeQuery( - String.format( - "source=%s | stats first(account_number), last(balance), first(age)", - TEST_INDEX_BANK)); + executeQuery(BANK_STREAM + " | stats first(account_number), last(balance), first(age)"); verifySchema( actual, schema("first(account_number)", "bigint"), @@ -478,11 +488,17 @@ public void testFirstLastDifferentFields() throws IOException { verifyDataRows(actual, rows(1L, 48086L, 32L)); } + // The text-field, nested-field, alias and script cases below must stay index-backed because they + // exercise field-access paths that makeresults cannot reproduce. They are made deterministic by + // narrowing to a single candidate document per output cell, so the selected row no longer depends + // on shard-local document order. @Test public void testFirstAggregationOnTextField() throws IOException { JSONObject actual = executeQuery( - String.format("source=%s | stats first(employer), first(email)", TEST_INDEX_BANK)); + String.format( + "source=%s | where account_number = 1 | stats first(employer), first(email)", + TEST_INDEX_BANK)); verifySchema(actual, schema("first(employer)", "string"), schema("first(email)", "string")); verifyDataRows(actual, rows("Pyrami", "amberduke@pyrami.com")); } @@ -491,9 +507,11 @@ public void testFirstAggregationOnTextField() throws IOException { public void testLastAggregationOnTextField() throws IOException { JSONObject actual = executeQuery( - String.format("source=%s | stats last(employer), first(email)", TEST_INDEX_BANK)); + String.format( + "source=%s | where account_number = 32 | stats last(employer), first(email)", + TEST_INDEX_BANK)); verifySchema(actual, schema("last(employer)", "string"), schema("first(email)", "string")); - verifyDataRows(actual, rows("Quailcom", "amberduke@pyrami.com")); + verifyDataRows(actual, rows("Quailcom", "dillardmcpherson@quailcom.com")); } @Test @@ -501,7 +519,9 @@ public void testFirstLastByGroupOnTextField() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | stats first(employer), last(email) by gender", TEST_INDEX_BANK)); + "source=%s | where account_number in (1, 13) | stats first(employer), last(email)" + + " by gender", + TEST_INDEX_BANK)); verifySchema( actual, schema("first(employer)", "string"), @@ -509,8 +529,8 @@ public void testFirstLastByGroupOnTextField() throws IOException { schema("gender", "string")); verifyDataRows( actual, - rows("Quility", "dillardmcpherson@quailcom.com", "F"), - rows("Pyrami", "elinorratliff@scentric.com", "M")); + rows("Quility", "nanettebates@quility.com", "F"), + rows("Pyrami", "amberduke@pyrami.com", "M")); } @Test @@ -518,7 +538,8 @@ public void testFirstLastWithOtherAggregationsOnTextField() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | stats first(employer), last(email), count(), avg(age) by gender", + "source=%s | where account_number in (1, 13) | stats first(employer), last(email)," + + " count(), avg(age) by gender", TEST_INDEX_BANK)); verifySchema( actual, @@ -529,16 +550,14 @@ public void testFirstLastWithOtherAggregationsOnTextField() throws IOException { schema("gender", "string")); verifyDataRows( actual, - rows("Quility", "dillardmcpherson@quailcom.com", 3, 33.666666666666664, "F"), - rows("Pyrami", "elinorratliff@scentric.com", 4, 34.25, "M")); + rows("Quility", "nanettebates@quility.com", 1, 28.0, "F"), + rows("Pyrami", "amberduke@pyrami.com", 1, 32.0, "M")); } @Test public void testFirstLastMixedFields() throws IOException { JSONObject actual = - executeQuery( - String.format( - "source=%s | stats first(employer), last(balance), first(age)", TEST_INDEX_BANK)); + executeQuery(BANK_STREAM + " | stats first(employer), last(balance), first(age)"); verifySchema( actual, schema("first(employer)", "string"), @@ -551,10 +570,12 @@ public void testFirstLastMixedFields() throws IOException { public void testFirstLastWithBirthdate() throws IOException { JSONObject actual = executeQuery( - String.format("source=%s | stats first(birthdate), last(birthdate)", TEST_INDEX_BANK)); + String.format( + "source=%s | where account_number = 1 | stats first(birthdate), last(birthdate)", + TEST_INDEX_BANK)); verifySchema( actual, schema("first(birthdate)", "timestamp"), schema("last(birthdate)", "timestamp")); - verifyDataRows(actual, rows("2017-10-23 00:00:00", "2018-08-11 00:00:00")); + verifyDataRows(actual, rows("2017-10-23 00:00:00", "2017-10-23 00:00:00")); } @Test @@ -562,8 +583,8 @@ public void testFirstLastBirthdateByGender() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | stats first(birthdate) as first_bd, last(birthdate) as last_bd by" - + " gender", + "source=%s | where account_number in (1, 13) | stats first(birthdate) as first_bd," + + " last(birthdate) as last_bd by gender", TEST_INDEX_BANK)); verifySchema( actual, @@ -572,8 +593,8 @@ public void testFirstLastBirthdateByGender() throws IOException { schema("gender", "string")); verifyDataRows( actual, - rows("2017-10-23 00:00:00", "2018-06-27 00:00:00", "M"), - rows("2018-06-23 00:00:00", "2018-08-11 00:00:00", "F")); + rows("2017-10-23 00:00:00", "2017-10-23 00:00:00", "M"), + rows("2018-06-23 00:00:00", "2018-06-23 00:00:00", "F")); } @Test @@ -581,8 +602,8 @@ public void testFirstLastBirthdateWithOtherFields() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | stats first(firstname), first(birthdate), last(lastname)," - + " last(birthdate) by gender", + "source=%s | where account_number in (1, 13) | stats first(firstname)," + + " first(birthdate), last(lastname), last(birthdate) by gender", TEST_INDEX_BANK)); verifySchema( actual, @@ -593,8 +614,8 @@ public void testFirstLastBirthdateWithOtherFields() throws IOException { schema("gender", "string")); verifyDataRows( actual, - rows("Amber JOHnny", "2017-10-23 00:00:00", "Ratliff", "2018-06-27 00:00:00", "M"), - rows("Nanette", "2018-06-23 00:00:00", "Mcpherson", "2018-08-11 00:00:00", "F")); + rows("Amber JOHnny", "2017-10-23 00:00:00", "Duke Willmington", "2017-10-23 00:00:00", "M"), + rows("Nanette", "2018-06-23 00:00:00", "Bates", "2018-06-23 00:00:00", "F")); } @Test @@ -602,20 +623,29 @@ public void testFirstLastWithTimestamp() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | stats first(timestamp), last(timestamp)", TEST_INDEX_TIME_DATA)); + "source=%s | where value = 8945 | stats first(timestamp), last(timestamp)", + TEST_INDEX_TIME_DATA)); verifySchema( actual, schema("first(timestamp)", "timestamp"), schema("last(timestamp)", "timestamp")); - verifyDataRows(actual, rows("2025-07-28 00:15:23", "2025-08-01 03:47:41")); + verifyDataRows(actual, rows("2025-07-28 00:15:23", "2025-07-28 00:15:23")); } @Test public void testFirstLastTimestampByCategory() throws IOException { - JSONObject actual = - executeQuery( - String.format( - "source=%s | stats first(timestamp) as first_ts, last(timestamp) as last_ts by" - + " category", - TEST_INDEX_TIME_DATA)); + // Two rows per category, early row first, so first_ts and last_ts differ per group. The values + // are the same ones the index-backed query produced on a single shard. + JSONObject actual = + executeQuery( + "makeresults format=csv data='category:string,ts:string\\n" + + "A,2025-07-28 00:15:23\\n" + + "B,2025-07-28 01:42:15\\n" + + "C,2025-07-28 02:28:45\\n" + + "D,2025-07-28 04:33:10\\n" + + "B,2025-08-01 01:14:11\\n" + + "C,2025-08-01 02:00:56\\n" + + "D,2025-08-01 00:27:26\\n" + + "A,2025-08-01 03:47:41' | eval timestamp = cast(ts as timestamp) | stats" + + " first(timestamp) as first_ts, last(timestamp) as last_ts by category"); verifySchema( actual, schema("first_ts", "timestamp"), @@ -634,7 +664,8 @@ public void testFirstLastTimestampWithValue() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | stats first(value), first(timestamp), last(value), last(timestamp)", + "source=%s | where value = 8945 | stats first(value), first(timestamp)," + + " last(value), last(timestamp)", TEST_INDEX_TIME_DATA)); verifySchema( actual, @@ -642,19 +673,19 @@ public void testFirstLastTimestampWithValue() throws IOException { schema("first(timestamp)", "timestamp"), schema("last(value)", "int"), schema("last(timestamp)", "timestamp")); - verifyDataRows(actual, rows(8945, "2025-07-28 00:15:23", 8762, "2025-08-01 03:47:41")); + verifyDataRows(actual, rows(8945, "2025-07-28 00:15:23", 8945, "2025-07-28 00:15:23")); } @Test public void testFirstLastWithNullValues() throws IOException { + // Nulls are skipped, so the result is the first and last non-null value in stream order. The + // balance column mirrors the bank_with_null_values fixture, where accounts 6, 20 and 25 have no + // balance. JSONObject actual = executeQuery( - String.format( - "source=%s | stats first(balance) as first_bal, last(balance) as last_bal", - TEST_INDEX_BANK_WITH_NULL_VALUES)); + "makeresults format=csv data='balance:long\\n39225\\n\\n32838\\n4180\\n\\n\\n48086' |" + + " stats first(balance) as first_bal, last(balance) as last_bal"); verifySchema(actual, schema("first_bal", "bigint"), schema("last_bal", "bigint")); - // Note: Current implementation skips nulls, so we expect first and last non-null values - // This test verifies current behavior - may need to change based on requirements verifyDataRows(actual, rows(39225L, 48086L)); } @@ -1132,9 +1163,7 @@ public void testSimpleTwoLevelStats() throws IOException { @Test public void testTake() throws IOException { - JSONObject actual = - executeQuery( - String.format("source=%s | stats take(firstname, 2) as take", TEST_INDEX_BANK)); + JSONObject actual = executeQuery(BANK_STREAM + " | stats take(firstname, 2) as take"); verifySchema(actual, schema("take", "array")); verifyDataRows(actual, rows(List.of("Amber JOHnny", "Hattie"))); } @@ -1538,18 +1567,23 @@ public void testFirstLastWithSimpleField() throws IOException { // This should work - testing simple field first JSONObject actual = executeQuery( - String.format("source=%s | stats first(severityNumber)", TEST_INDEX_TELEMETRY)); + String.format( + "source=%s | where `resource.attributes.telemetry.sdk.version` = 10 | stats" + + " first(severityNumber)", + TEST_INDEX_TELEMETRY)); verifySchema(actual, schema("first(severityNumber)", "int")); verifyDataRows(actual, rows(9)); } @Test public void testFirstLastWithDeepNestedField() throws IOException { - // This test should now work with the fix for ClassCastException + // Narrowed to the single document with version 10 so the selected row does not depend on + // shard-local document order. The nested-field access path is still exercised. JSONObject actual = executeQuery( String.format( - "source=%s | stats first(`resource.attributes.telemetry.sdk.language`)", + "source=%s | where `resource.attributes.telemetry.sdk.version` = 10 | stats" + + " first(`resource.attributes.telemetry.sdk.language`)", TEST_INDEX_TELEMETRY)); verifySchema(actual, schema("first(`resource.attributes.telemetry.sdk.language`)", "string")); verifyDataRows(actual, rows("java")); @@ -1557,11 +1591,11 @@ public void testFirstLastWithDeepNestedField() throws IOException { @Test public void testLastWithDeepNestedField() throws IOException { - // This test should now work with the fix for ClassCastException JSONObject actual = executeQuery( String.format( - "source=%s | stats last(`resource.attributes.telemetry.sdk.language`)", + "source=%s | where `resource.attributes.telemetry.sdk.version` = 14 | stats" + + " last(`resource.attributes.telemetry.sdk.language`)", TEST_INDEX_TELEMETRY)); verifySchema(actual, schema("last(`resource.attributes.telemetry.sdk.language`)", "string")); verifyDataRows(actual, rows("rust")); @@ -1569,11 +1603,12 @@ public void testLastWithDeepNestedField() throws IOException { @Test public void testFirstLastWithDeepNestedFieldByGroup() throws IOException { - // This test should now work with the fix for ClassCastException + // Versions 10, 11 and 13 give exactly one document per severityNumber group. JSONObject actual = executeQuery( String.format( - "source=%s | stats first(`resource.attributes.telemetry.sdk.language`) by" + "source=%s | where `resource.attributes.telemetry.sdk.version` in (10, 11, 13) |" + + " stats first(`resource.attributes.telemetry.sdk.language`) by" + " severityNumber", TEST_INDEX_TELEMETRY)); verifySchema( @@ -1688,15 +1723,17 @@ public void testMinMaxIntegerNestedFieldsByGroup() throws IOException { @Test public void testFirstLastWithIntegerNestedField() throws IOException { - // Test first/last with deeply nested integer fields + // Test first/last with deeply nested integer fields. Narrowed to the single severityNumber 16 + // document so the selection is independent of shard-local document order. JSONObject actual = executeQuery( String.format( - "source=%s | stats first(`resource.attributes.telemetry.sdk.version`) as first_ver," + "source=%s | where severityNumber = 16 | stats" + + " first(`resource.attributes.telemetry.sdk.version`) as first_ver," + " last(`resource.attributes.telemetry.sdk.version`) as last_ver", TEST_INDEX_TELEMETRY)); verifySchema(actual, schema("first_ver", "int"), schema("last_ver", "int")); - verifyDataRows(actual, rows(10, 14)); + verifyDataRows(actual, rows(13, 13)); } @Test @@ -1705,7 +1742,8 @@ public void testFirstLastWithBooleanNestedField() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | stats first(`resource.attributes.telemetry.sdk.enabled`) as" + "source=%s | where severityNumber = 9 | stats" + + " first(`resource.attributes.telemetry.sdk.enabled`) as" + " first_enabled, last(`resource.attributes.telemetry.sdk.enabled`) as" + " last_enabled", TEST_INDEX_TELEMETRY)); @@ -1743,11 +1781,13 @@ public void testMinMaxWithBooleanNestedField() throws IOException { @Test public void testBooleanNestedFieldByGroup() throws IOException { - // Test boolean nested fields with grouping by other fields + // Test boolean nested fields with grouping by other fields. Versions 10, 11 and 13 give exactly + // one document per severityNumber group, so first() does not depend on document order. JSONObject actual = executeQuery( String.format( - "source=%s | stats count() as cnt," + "source=%s | where `resource.attributes.telemetry.sdk.version` in (10, 11, 13) |" + + " stats count() as cnt," + " first(`resource.attributes.telemetry.sdk.enabled`) as enabled by" + " severityNumber", TEST_INDEX_TELEMETRY)); @@ -1756,19 +1796,21 @@ public void testBooleanNestedFieldByGroup() throws IOException { schema("cnt", "bigint"), schema("enabled", "boolean"), schema("severityNumber", "int")); - // severityNumber 9: java (true), javascript (true) -> 2 records, first is true - // severityNumber 12: python (false), rust (true) -> 2 records, first is false - // severityNumber 16: go (false) -> 1 record, first is false - verifyDataRows(actual, rows(2L, true, 9), rows(2L, false, 12), rows(1L, false, 16)); + // severityNumber 9: java (true); severityNumber 12: python (false); severityNumber 16: go + // (false) + verifyDataRows(actual, rows(1L, true, 9), rows(1L, false, 12), rows(1L, false, 16)); } @Test public void testMixedTypesNestedFieldAggregations() throws IOException { - // Test aggregating multiple nested field types in one query + // Test aggregating multiple nested field types in one query. Narrowed to the single + // severityNumber 16 document so first() does not depend on document order. min/max across + // multiple documents are covered by the dedicated nested min/max tests above. JSONObject actual = executeQuery( String.format( - "source=%s | stats min(`resource.attributes.telemetry.sdk.version`) as min_ver," + "source=%s | where severityNumber = 16 | stats" + + " min(`resource.attributes.telemetry.sdk.version`) as min_ver," + " max(`resource.attributes.telemetry.sdk.version`) as max_ver," + " min(`resource.attributes.telemetry.sdk.enabled`) as min_enabled," + " max(`resource.attributes.telemetry.sdk.enabled`) as max_enabled," @@ -1781,7 +1823,7 @@ public void testMixedTypesNestedFieldAggregations() throws IOException { schema("min_enabled", "boolean"), schema("max_enabled", "boolean"), schema("first_lang", "string")); - verifyDataRows(actual, rows(10, 14, false, true, "java")); + verifyDataRows(actual, rows(13, 13, false, false, "go")); } @Test @@ -1789,21 +1831,22 @@ public void testTextTypeWithLastFirstTakeAggregation() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | stats take(address, 2), last(address), first(address), " + "source=%s | where account_number = 1 | stats take(address, 2), last(address)," + + " first(address), " + "take(state, 2), last(state), first(state), " + "take(balance, 2), last(balance), first(balance)", TEST_INDEX_BANK)); verifyDataRows( actual, rows( - List.of("880 Holmes Lane", "671 Bristol Street"), - "702 Quentin Street", + List.of("880 Holmes Lane"), "880 Holmes Lane", - List.of("IL", "TN"), - "IN", + "880 Holmes Lane", + List.of("IL"), + "IL", "IL", - List.of(39225, 5686), - 48086, + List.of(39225), + 39225, 39225)); } @@ -1812,22 +1855,23 @@ public void testScriptWithLastFirstTakeAggregation() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | eval new_address = upper(address), new_state = lower(state)," - + " new_balance = balance * 10 | stats take(new_address, 2), last(new_address)," - + " first(new_address), take(new_state, 2), last(new_state), first(new_state)," - + " take(new_balance, 2), last(new_balance), first(new_balance)", + "source=%s | where account_number = 1 | eval new_address = upper(address)," + + " new_state = lower(state), new_balance = balance * 10 | stats" + + " take(new_address, 2), last(new_address), first(new_address)," + + " take(new_state, 2), last(new_state), first(new_state), take(new_balance," + + " 2), last(new_balance), first(new_balance)", TEST_INDEX_BANK)); verifyDataRows( actual, rows( - List.of("880 HOLMES LANE", "671 BRISTOL STREET"), - "702 QUENTIN STREET", + List.of("880 HOLMES LANE"), "880 HOLMES LANE", - List.of("il", "tn"), - "in", + "880 HOLMES LANE", + List.of("il"), + "il", "il", - List.of(392250, 56860), - 480860, + List.of(392250), + 392250, 392250)); } From 2c34ab4f2a7656de32e5f8df996b1ceab701a33d Mon Sep 17 00:00:00 2001 From: Eric Wei Date: Fri, 28 Aug 2026 20:50:01 +0000 Subject: [PATCH 2/2] test(integ-test): preserve multi-document first/last coverage Keep deterministic makeresults tests for exact stream-position semantics, while restoring multi-document index queries with membership assertions that do not assume shard order. This retains alias, nested, grouped, eval, TAKE, pushdown, and no-pushdown coverage without imposing deterministic FIRST/LAST results across shards. Signed-off-by: Eric Wei --- .../CalciteAliasFieldAggregationIT.java | 124 ++++++---- .../remote/CalcitePPLAggregationIT.java | 215 ++++++++++++++++++ 2 files changed, 294 insertions(+), 45 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java index 63d2a433c5f..08180ed640f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java @@ -13,7 +13,9 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchemaInOrder; import java.io.IOException; -import java.util.List; +import java.util.HashSet; +import java.util.Set; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.client.Request; @@ -109,22 +111,18 @@ public void testMinMaxWithNumericAliasField() throws IOException { public void testFirstWithAliasField() throws IOException { JSONObject actual = executeQuery( - String.format( - "source=%s | where value = 100 | sort @timestamp | stats FIRST(@timestamp)", - TEST_ALIAS_BUG)); + String.format("source=%s | sort @timestamp | stats FIRST(@timestamp)", TEST_ALIAS_BUG)); verifySchema(actual, schema("FIRST(@timestamp)", "timestamp")); - verifyDataRows(actual, rows("2024-01-01 10:00:00")); + assertTimestampMember(actual.getJSONArray("datarows").getJSONArray(0).get(0)); } @Test public void testLastWithAliasField() throws IOException { JSONObject actual = executeQuery( - String.format( - "source=%s | where value = 300 | sort @timestamp | stats LAST(@timestamp)", - TEST_ALIAS_BUG)); + String.format("source=%s | sort @timestamp | stats LAST(@timestamp)", TEST_ALIAS_BUG)); verifySchema(actual, schema("LAST(@timestamp)", "timestamp")); - verifyDataRows(actual, rows("2024-01-03 10:00:00")); + assertTimestampMember(actual.getJSONArray("datarows").getJSONArray(0).get(0)); } @Test @@ -132,10 +130,9 @@ public void testTakeWithAliasField() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | where value = 100 | sort @timestamp | stats TAKE(@timestamp, 2)", - TEST_ALIAS_BUG)); + "source=%s | sort @timestamp | stats TAKE(@timestamp, 2)", TEST_ALIAS_BUG)); verifySchema(actual, schema("TAKE(@timestamp, 2)", "array")); - verifyDataRows(actual, rows(List.of("2024-01-01 10:00:00"))); + assertTimestampTakeMembers(actual.getJSONArray("datarows").getJSONArray(0).getJSONArray(0)); } @Test @@ -165,42 +162,79 @@ public void testUnaffectedAggregationsWithAliasFields() throws IOException { @Test public void testAliasTypeWithLastFirstTakeLatestEarliestAggregation() throws IOException { - // take/first/last select by document order, which is not defined across shards, so they are - // asserted against a single narrowed document. latest/earliest are ordered by the time field - // and - // stay deterministic, so they keep asserting over the full dataset. JSONObject actual = executeQuery( String.format( - "source=%s | where original_col = 1 | stats take(original_text, 2)," - + " last(original_text), first(original_text), take(alias_text, 2)," - + " last(alias_text), first(alias_text), take(original_col, 2)," - + " last(original_col), first(original_col), take(alias_col, 2)," - + " last(alias_col), first(alias_col)", - TEST_INDEX_ALIAS)); - verifyDataRows( - actual, - rows( - List.of("a b c"), - "a b c", - "a b c", - List.of("a b c"), - "a b c", - "a b c", - List.of(1), - 1, - 1, - List.of(1), - 1, - 1)); - - JSONObject ordered = - executeQuery( - String.format( - "source=%s | stats latest(original_col), earliest(original_col)," - + " latest(alias_col), earliest(alias_col), latest(original_text)," - + " earliest(original_text), latest(alias_text), earliest(alias_text)", + "source=%s | stats take(original_text, 2), last(original_text)," + + " first(original_text), take(alias_text, 2), last(alias_text)," + + " first(alias_text), take(original_col, 2), last(original_col)," + + " first(original_col), take(alias_col, 2), last(alias_col), first(alias_col)," + + " latest(original_col), earliest(original_col), latest(alias_col)," + + " earliest(alias_col), latest(original_text), earliest(original_text)," + + " latest(alias_text), earliest(alias_text)", TEST_INDEX_ALIAS)); - verifyDataRows(ordered, rows(3, 1, 3, 1, "x y z", "a b c", "x y z", "a b c")); + + JSONArray row = actual.getJSONArray("datarows").getJSONArray(0); + Set texts = Set.of("a b c", "d e f", "x y z"); + Set numbers = Set.of(1, 2, 3); + assertTakeMembers(row.getJSONArray(0), 2, texts); + assertMember(row.get(1), texts); + assertMember(row.get(2), texts); + assertTakeMembers(row.getJSONArray(3), 2, texts); + assertMember(row.get(4), texts); + assertMember(row.get(5), texts); + assertTakeMembers(row.getJSONArray(6), 2, numbers); + assertMember(row.get(7), numbers); + assertMember(row.get(8), numbers); + assertTakeMembers(row.getJSONArray(9), 2, numbers); + assertMember(row.get(10), numbers); + assertMember(row.get(11), numbers); + + assertEquals(3, row.getInt(12)); + assertEquals(1, row.getInt(13)); + assertEquals(3, row.getInt(14)); + assertEquals(1, row.getInt(15)); + assertEquals("x y z", row.getString(16)); + assertEquals("a b c", row.getString(17)); + assertEquals("x y z", row.getString(18)); + assertEquals("a b c", row.getString(19)); + } + + private static void assertMember(Object actual, Set expectedValues) { + assertTrue( + "Expected one of " + expectedValues + " but got " + actual, + expectedValues.contains(actual)); + } + + private static void assertTakeMembers(JSONArray actual, int expectedSize, Set expectedValues) { + assertEquals(expectedSize, actual.length()); + Set distinct = new HashSet<>(); + for (int i = 0; i < actual.length(); i++) { + Object value = actual.get(i); + assertMember(value, expectedValues); + distinct.add(value); + } + assertEquals(expectedSize, distinct.size()); + } + + private static String normalizeTimestamp(Object value) { + return value.toString().replace('T', ' ').replace(".000Z", "").replace("Z", ""); + } + + private static void assertTimestampMember(Object actual) { + assertMember( + normalizeTimestamp(actual), + Set.of("2024-01-01 10:00:00", "2024-01-02 10:00:00", "2024-01-03 10:00:00")); + } + + private static void assertTimestampTakeMembers(JSONArray actual) { + assertEquals(2, actual.length()); + Set normalized = new HashSet<>(); + for (int i = 0; i < actual.length(); i++) { + String value = normalizeTimestamp(actual.get(i)); + assertTimestampMember(value); + normalized.add(value); + } + assertEquals(2, normalized.size()); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java index f839491c6e8..06bf8456855 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java @@ -26,7 +26,10 @@ import java.io.IOException; import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.client.Request; @@ -57,6 +60,23 @@ public class CalcitePPLAggregationIT extends PPLIntegTestCase { + "25,Virginia,Ayala,Filodyne,F,39,40540\\n" + "32,Dillard,Mcpherson,Quailcom,F,34,48086'"; + private static void assertMember(Object actual, Set expectedValues) { + assertTrue( + "Expected one of " + expectedValues + " but got " + actual, + expectedValues.contains(actual)); + } + + private static void assertTakeMembers(JSONArray actual, int expectedSize, Set expectedValues) { + assertEquals(expectedSize, actual.length()); + Set distinct = new HashSet<>(); + for (int i = 0; i < actual.length(); i++) { + Object value = actual.get(i); + assertMember(value, expectedValues); + distinct.add(value); + } + assertEquals(expectedSize, distinct.size()); + } + @Override public void init() throws Exception { super.init(); @@ -709,6 +729,201 @@ public void testFirstLastWithNullValuesByGroup() throws IOException { rows(null, null, 36)); // balance is null for age 36 } + @Test + public void testIndexBackedFirstLastValuesBelongToSourceAcrossShards() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | stats first(firstname), last(firstname), first(employer), last(email)," + + " first(age), last(balance), first(birthdate), last(birthdate)", + TEST_INDEX_BANK)); + verifySchema( + actual, + schema("first(firstname)", "string"), + schema("last(firstname)", "string"), + schema("first(employer)", "string"), + schema("last(email)", "string"), + schema("first(age)", "int"), + schema("last(balance)", "bigint"), + schema("first(birthdate)", "timestamp"), + schema("last(birthdate)", "timestamp")); + + JSONArray row = actual.getJSONArray("datarows").getJSONArray(0); + assertMember( + row.get(0), + Set.of("Amber JOHnny", "Hattie", "Nanette", "Dale", "Elinor", "Virginia", "Dillard")); + assertMember( + row.get(1), + Set.of("Amber JOHnny", "Hattie", "Nanette", "Dale", "Elinor", "Virginia", "Dillard")); + assertMember( + row.get(2), + Set.of("Pyrami", "Netagy", "Quility", "Boink", "Scentric", "Filodyne", "Quailcom")); + assertMember( + row.get(3), + Set.of( + "amberduke@pyrami.com", + "hattiebond@netagy.com", + "nanettebates@quility.com", + "daleadams@boink.com", + "elinorratliff@scentric.com", + "virginiaayala@filodyne.com", + "dillardmcpherson@quailcom.com")); + assertMember(row.getInt(4), Set.of(28, 32, 33, 34, 36, 39)); + assertMember(row.getLong(5), Set.of(4180L, 5686L, 16418L, 32838L, 39225L, 40540L, 48086L)); + assertMember( + row.getString(6), + Set.of( + "2017-10-23 00:00:00", + "2017-11-20 00:00:00", + "2018-06-23 00:00:00", + "2018-06-27 00:00:00", + "2018-08-11 00:00:00", + "2018-08-19 00:00:00", + "2018-11-13 23:33:20")); + assertMember( + row.getString(7), + Set.of( + "2017-10-23 00:00:00", + "2017-11-20 00:00:00", + "2018-06-23 00:00:00", + "2018-06-27 00:00:00", + "2018-08-11 00:00:00", + "2018-08-19 00:00:00", + "2018-11-13 23:33:20")); + } + + @Test + public void testIndexBackedFirstLastByGroupRemainWithinGroupAcrossShards() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | stats first(firstname), last(lastname), first(employer), last(email)," + + " count(), avg(age) by gender", + TEST_INDEX_BANK)); + verifySchema( + actual, + schema("first(firstname)", "string"), + schema("last(lastname)", "string"), + schema("first(employer)", "string"), + schema("last(email)", "string"), + schema("count()", "bigint"), + schema("avg(age)", "double"), + schema("gender", "string")); + + JSONArray rows = actual.getJSONArray("datarows"); + assertEquals(2, rows.length()); + for (int i = 0; i < rows.length(); i++) { + JSONArray row = rows.getJSONArray(i); + if ("F".equals(row.getString(6))) { + assertMember(row.get(0), Set.of("Nanette", "Virginia", "Dillard")); + assertMember(row.get(1), Set.of("Bates", "Ayala", "Mcpherson")); + assertMember(row.get(2), Set.of("Quility", "Filodyne", "Quailcom")); + assertMember( + row.get(3), + Set.of( + "nanettebates@quility.com", + "virginiaayala@filodyne.com", + "dillardmcpherson@quailcom.com")); + assertEquals(3L, row.getLong(4)); + assertEquals(33.666666666666664, row.getDouble(5), 0.0); + } else { + assertEquals("M", row.getString(6)); + assertMember(row.get(0), Set.of("Amber JOHnny", "Hattie", "Dale", "Elinor")); + assertMember(row.get(1), Set.of("Duke Willmington", "Bond", "Adams", "Ratliff")); + assertMember(row.get(2), Set.of("Pyrami", "Netagy", "Boink", "Scentric")); + assertMember( + row.get(3), + Set.of( + "amberduke@pyrami.com", + "hattiebond@netagy.com", + "daleadams@boink.com", + "elinorratliff@scentric.com")); + assertEquals(4L, row.getLong(4)); + assertEquals(34.25, row.getDouble(5), 0.0); + } + } + } + + @Test + public void testIndexBackedNestedFirstLastRemainWithinGroupAcrossShards() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | stats first(`resource.attributes.telemetry.sdk.language`) as lang," + + " last(`resource.attributes.telemetry.sdk.version`) as version," + + " first(`resource.attributes.telemetry.sdk.enabled`) as enabled, count() as" + + " cnt by severityNumber", + TEST_INDEX_TELEMETRY)); + verifySchema( + actual, + schema("lang", "string"), + schema("version", "int"), + schema("enabled", "boolean"), + schema("cnt", "bigint"), + schema("severityNumber", "int")); + + JSONArray rows = actual.getJSONArray("datarows"); + assertEquals(3, rows.length()); + for (int i = 0; i < rows.length(); i++) { + JSONArray row = rows.getJSONArray(i); + switch (row.getInt(4)) { + case 9 -> { + assertMember(row.get(0), Set.of("java", "javascript")); + assertMember(row.getInt(1), Set.of(10, 12)); + assertEquals(true, row.getBoolean(2)); + assertEquals(2L, row.getLong(3)); + } + case 12 -> { + assertMember(row.get(0), Set.of("python", "rust")); + assertMember(row.getInt(1), Set.of(11, 14)); + assertMember(row.getBoolean(2), Set.of(false, true)); + assertEquals(2L, row.getLong(3)); + } + case 16 -> { + assertEquals("go", row.getString(0)); + assertEquals(13, row.getInt(1)); + assertEquals(false, row.getBoolean(2)); + assertEquals(1L, row.getLong(3)); + } + default -> fail("Unexpected severityNumber " + row.getInt(4)); + } + } + } + + @Test + public void testIndexBackedTakeFirstLastWithEvalReturnSourceMembersAcrossShards() + throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval new_address = upper(address), new_state = lower(state)," + + " new_balance = balance * 10 | stats take(new_address, 2), last(new_address)," + + " first(new_address), take(new_state, 2), last(new_state), first(new_state)," + + " take(new_balance, 2), last(new_balance), first(new_balance)", + TEST_INDEX_BANK)); + JSONArray row = actual.getJSONArray("datarows").getJSONArray(0); + Set addresses = + Set.of( + "880 HOLMES LANE", + "671 BRISTOL STREET", + "789 MADISON STREET", + "467 HUTCHINSON COURT", + "282 KINGS PLACE", + "171 PUTNAM AVENUE", + "702 QUENTIN STREET"); + Set states = Set.of("il", "tn", "va", "md", "wa", "pa", "in"); + Set balances = Set.of(392250, 56860, 328380, 41800, 164180, 405400, 480860); + assertTakeMembers(row.getJSONArray(0), 2, addresses); + assertMember(row.get(1), addresses); + assertMember(row.get(2), addresses); + assertTakeMembers(row.getJSONArray(3), 2, states); + assertMember(row.get(4), states); + assertMember(row.get(5), states); + assertTakeMembers(row.getJSONArray(6), 2, balances); + assertMember(row.get(7), balances); + assertMember(row.get(8), balances); + } + @Test public void testAvgBySpanAndFields() throws IOException { JSONObject actual =