Skip to content

Commit 25d9c20

Browse files
not-napoleonelasticsearchmachine
not-napoleon
and
elasticsearchmachine
authored
Esql - Fix lucene push down behavior when a range contains nanos and millis (#125595) (#125618)
Follow up to #125345. If the query contained both a nanos and a millis comparison, we were formatting the dates incorrectly for the lucene push down. This PR adds a test and a fix for that case. --------- Co-authored-by: elasticsearchmachine <infra-root+elasticsearchmachine@elastic.co>
1 parent bb39781 commit 25d9c20

File tree

4 files changed

+133
-10
lines changed

4 files changed

+133
-10
lines changed

‎docs/changelog/125595.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 125595
2+
summary: Esql - Fix lucene push down behavior when a range contains nanos and millis
3+
area: ES|QL
4+
type: bug
5+
issues: []

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1753,3 +1753,110 @@ nanos:date_nanos
17531753
2023-10-23T12:15:03.360103847Z
17541754
2023-10-23T12:15:03.360103847Z
17551755
;
1756+
1757+
Range Without Included Endpoints
1758+
required_capability: to_date_nanos
1759+
required_capability: fix_date_nanos_lucene_pushdown_bug
1760+
1761+
FROM date_nanos
1762+
| WHERE millis > "2020-01-01"
1763+
| WHERE nanos > to_datenanos("2023-10-23T12:15:03.360103847") AND nanos < to_datenanos("2023-10-23T13:53:55.832987654Z")
1764+
| KEEP nanos;
1765+
ignoreOrder:true
1766+
warningRegex:Line 3:67: evaluation of \[nanos < to_datenanos\(\\\"2023-10-23T13:53:55\.832987654Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
1767+
warningRegex:Line 3:67: java.lang.IllegalArgumentException: single-value function encountered multi-value
1768+
warningRegex:Line 3:9: evaluation of \[nanos > to_datenanos\(\\\"2023-10-23T12:15:03\.360103847\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
1769+
warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
1770+
1771+
nanos:date_nanos
1772+
2023-10-23T13:52:55.015787878Z
1773+
2023-10-23T13:51:54.732102837Z
1774+
2023-10-23T13:33:34.937193000Z
1775+
2023-10-23T12:27:28.948000000Z
1776+
;
1777+
1778+
Range Without Included Endpoints with implicit casting
1779+
required_capability: to_date_nanos
1780+
required_capability: fix_date_nanos_lucene_pushdown_bug
1781+
1782+
FROM date_nanos
1783+
| WHERE millis > "2020-01-01"
1784+
| WHERE nanos > "2023-10-23T12:15:03.360103847" AND nanos < "2023-10-23T13:53:55.832987654Z"
1785+
| KEEP nanos;
1786+
ignoreOrder:true
1787+
warningRegex:Line 3:53: evaluation of \[nanos < \\\"2023-10-23T13:53:55\.832987654Z\\\"\] failed, treating result as null\. Only first 20 failures recorded\.
1788+
warningRegex:Line 3:53: java.lang.IllegalArgumentException: single-value function encountered multi-value
1789+
warningRegex:Line 3:9: evaluation of \[nanos > \\\"2023-10-23T12:15:03\.360103847\\\"\] failed, treating result as null\. Only first 20 failures recorded\.
1790+
warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
1791+
1792+
nanos:date_nanos
1793+
2023-10-23T13:52:55.015787878Z
1794+
2023-10-23T13:51:54.732102837Z
1795+
2023-10-23T13:33:34.937193000Z
1796+
2023-10-23T12:27:28.948000000Z
1797+
;
1798+
1799+
Range With Now date math
1800+
required_capability: to_date_nanos
1801+
required_capability: fix_date_nanos_lucene_pushdown_bug
1802+
1803+
FROM date_nanos
1804+
| WHERE millis > "2020-01-01"
1805+
| WHERE nanos > TO_DATETIME("2023-10-23T12:27:28.948") AND nanos < now() - 1d
1806+
| KEEP nanos;
1807+
ignoreOrder:true
1808+
warningRegex:Line 3:60: evaluation of \[nanos < now\(\) - 1d\] failed, treating result as null\. Only first 20 failures recorded\.
1809+
warningRegex:Line 3:60: java.lang.IllegalArgumentException: single-value function encountered multi-value
1810+
warningRegex:Line 3:9: evaluation of \[nanos > TO_DATETIME\(\\\"2023-10-23T12:27:28\.948\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
1811+
warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
1812+
1813+
nanos:date_nanos
1814+
2023-10-23T13:55:01.543123456Z
1815+
2023-10-23T13:53:55.832987654Z
1816+
2023-10-23T13:52:55.015787878Z
1817+
2023-10-23T13:51:54.732102837Z
1818+
2023-10-23T13:33:34.937193000Z
1819+
;
1820+
1821+
Mixed Nanos Millis Range Compare
1822+
required_capability: to_date_nanos
1823+
required_capability: fix_date_nanos_lucene_pushdown_bug
1824+
required_capability: fix_date_nanos_mixed_range_pushdown_bug
1825+
1826+
FROM date_nanos
1827+
| WHERE millis > "2020-01-01"
1828+
| WHERE nanos > to_datenanos("2023-10-23T12:15:03.360103847") AND nanos < to_datetime("2023-10-23T13:53:55.832")
1829+
| KEEP nanos;
1830+
ignoreOrder:true
1831+
warningRegex:Line 3:67: evaluation of \[nanos < to_datetime\(\\\"2023-10-23T13:53:55\.832\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
1832+
warningRegex:Line 3:67: java.lang.IllegalArgumentException: single-value function encountered multi-value
1833+
warningRegex:Line 3:9: evaluation of \[nanos > to_datenanos\(\\\"2023-10-23T12:15:03\.360103847\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
1834+
warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
1835+
1836+
nanos:date_nanos
1837+
2023-10-23T13:52:55.015787878Z
1838+
2023-10-23T13:51:54.732102837Z
1839+
2023-10-23T13:33:34.937193000Z
1840+
2023-10-23T12:27:28.948000000Z
1841+
;
1842+
1843+
Mixed Nanos Millis Range Compare, millis field
1844+
required_capability: to_date_nanos
1845+
required_capability: fix_date_nanos_lucene_pushdown_bug
1846+
required_capability: fix_date_nanos_mixed_range_pushdown_bug
1847+
1848+
FROM date_nanos
1849+
| WHERE millis > to_datenanos("2023-10-23T12:15:03.360103847") AND millis < to_datetime("2023-10-23T13:53:55.832")
1850+
| KEEP nanos;
1851+
ignoreOrder:true
1852+
warningRegex:Line 3:67: evaluation of \[nanos < to_datetime\(\\\"2023-10-23T13:53:55\.832Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
1853+
warningRegex:Line 3:67: java.lang.IllegalArgumentException: single-value function encountered multi-value
1854+
warningRegex:Line 3:9: evaluation of \[nanos > to_datenanos\(\\\"2023-10-23T12:15:03\.360103847\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
1855+
warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
1856+
1857+
nanos:date_nanos
1858+
2023-10-23T13:52:55.015787878Z
1859+
2023-10-23T13:51:54.732102837Z
1860+
2023-10-23T13:33:34.937193000Z
1861+
2023-10-23T12:27:28.948000000Z
1862+
;

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -402,6 +402,11 @@ public enum Cap {
402402
* Indicates that https://github.com/elastic/elasticsearch/issues/125439 (incorrect lucene push down for date nanos) is fixed
403403
*/
404404
FIX_DATE_NANOS_LUCENE_PUSHDOWN_BUG(),
405+
/**
406+
* Fixes a bug where dates are incorrectly formatted if a where clause compares nanoseconds to both milliseconds and nanoseconds,
407+
* e.g. {@code WHERE millis > to_datenanos("2023-10-23T12:15:03.360103847") AND millis < to_datetime("2023-10-23T13:53:55.832")}
408+
*/
409+
FIX_DATE_NANOS_MIXED_RANGE_PUSHDOWN_BUG(),
405410
/**
406411
* DATE_PARSE supports reading timezones
407412
*/

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333

3434
import static java.util.Arrays.asList;
3535
import static org.elasticsearch.xpack.esql.core.expression.Foldables.valueOf;
36-
import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
3736
import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS;
3837
import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
3938
import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
@@ -42,9 +41,8 @@
4241
import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAsNumber;
4342
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_NANOS_FORMATTER;
4443
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER;
45-
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToString;
44+
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateWithTypeToString;
4645
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.ipToString;
47-
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.nanoTimeToString;
4846
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.versionToString;
4947

5048
// BETWEEN or range - is a mix of gt(e) AND lt(e)
@@ -217,16 +215,23 @@ private RangeQuery translate(TranslatorHandler handler) {
217215
String format = null;
218216

219217
DataType dataType = value.dataType();
220-
logger.trace("Translating Range into lucene query. dataType is [{}] upper is [{}] lower is [{}]", dataType, lower, upper);
221-
if (dataType == DataType.DATETIME && lower.dataType() == DATETIME && upper.dataType() == DATETIME) {
222-
l = dateTimeToString((Long) l);
223-
u = dateTimeToString((Long) u);
218+
logger.warn(
219+
"Translating Range into lucene query. dataType is [{}] upper is [{}<{}>] lower is [{}<{}>]",
220+
dataType,
221+
lower,
222+
lower.dataType(),
223+
upper,
224+
upper.dataType()
225+
);
226+
if (dataType == DataType.DATETIME) {
227+
l = dateWithTypeToString((Long) l, lower.dataType());
228+
u = dateWithTypeToString((Long) u, upper.dataType());
224229
format = DEFAULT_DATE_TIME_FORMATTER.pattern();
225230
}
226231

227-
if (dataType == DATE_NANOS && lower.dataType() == DATE_NANOS && upper.dataType() == DATE_NANOS) {
228-
l = nanoTimeToString((Long) l);
229-
u = nanoTimeToString((Long) u);
232+
if (dataType == DATE_NANOS) {
233+
l = dateWithTypeToString((Long) l, lower.dataType());
234+
u = dateWithTypeToString((Long) u, upper.dataType());
230235
format = DEFAULT_DATE_NANOS_FORMATTER.pattern();
231236
}
232237

@@ -258,6 +263,7 @@ private RangeQuery translate(TranslatorHandler handler) {
258263
u = unsignedLongAsNumber(ul);
259264
}
260265
}
266+
logger.warn("Building range query with format string [{}]", format);
261267
return new RangeQuery(source(), handler.nameOf(value), l, includeLower(), u, includeUpper(), format, zoneId);
262268
}
263269

0 commit comments

Comments
 (0)