Markduckworth/ppl mep (#8801)

* Multiple entry points for main package.

* MEP for firestore lite building

* Fix issue with external exports

* Refactor to support multiple entry points for the lite SDK. Fixes import issues and makes code organization across /pipelines and /lite/pipeliens

* Type and comment cleanup

* Make PipelineSource generic and remove useFluentPipelines

* Remove converter and placeholders for converter.

* cleanup: remove documentReferenceFactory

* Linting and formatting

* Update the typings export for pipelines and lite pipelines so that it auguments the Firestore and Query classes.

* Add missing Expr export

* Lint fixes

* TODO cleanup

* update api review

* Query to Proto with integration tests

* API tweaks from review. Selectable, FilterCondition, and Accumulator are abstract classes not interfaces. Removed extraneous Pipeline.sort overload

* Remove console.log statements from prune-dts.ts

* code cleanup in prune-dts

* Add missing imports to pipelines.ts

* If argument to FunctionExpr is a plain object or array, convert these to expression using map(...) or array(...)

* Add tests for evaluating expressions in arrays and maps

* implement new FunctionExpressions as standalone functions

* Fix typo and formatting in pipeline.test.ts

* Implement support for genericStage and genericFunction

* Implement new function expressions as methods on the Expr class.

* Pipeline test reorganization

* Implementing new stages Sample, Union, Unnest, Replace

* Rename FilterCondition to BooleanExpr

* Update functions with variadic params to match signatures of the backend

* Fix and implementation.

* Refactor AggregateFunction out of Expr

* Add ScalarExpr class and make Expr a base class of AggregateFunction

* fixing missing imports and renames

* Remove classes for individual function types

* Separate scalar expressions and aggregate functions into different class hierarchies

* Remove deleted Expr classes from the pipelines exports

* Replace not(fn()) with notFn() in the query to pipeline conversion

* Fix circular dependency between user_data_reader and expressions

* removing modular pipeline creation and removing fluent pipline execution. This was a decision from API review

* Change return type of execute from PipelineResult[] to PipelineSnapshot

* Renames for API review: FunctionExpr, BooleanExpr, params

* missing readonly

* Build and test fixes

* Replace Field.of() with field()

* Add missing export PipelineSnaphsot

* Replace Constant.of() with constant()

* Removed constant(...) overloads to create a map or array. Use the array or map function instead.

* Implemented and documented ascending|descending(fieldName) as a convenient way to create orderings for a specific field

* fix tests creating a constant map and array

* Remove genericFunction, genericAggregateFunction, and genericBooleanExpr. Replace usage with constructor

* Pipeline result timestamp testing

* Testing timestamp edge cases

* Query to pipeline conversion is performed from the PipelineSource class rather than the Query class

* Timestamp tests and fix pipeline tests for change in pipeline creation from a classic query

* test fix

* Refactor and rename all functions to match API proposal changes after API review.

* Fix build issues

* New integration util pipeline_export file to support the pipelines subpackage

* Rename several FunctionExpr standalone exports from <name>Function to <name>. Meets API design

* Removing the all-packages api report script because it is no longer needed

* Remove any from the public api surface

* lint fix

* Code cleanup

* FindNearest integration tests

* Filling in gaps in test coverage

* Implement Pipeline tests for the lite SDK

* Fixed a build issue with the lite SDK

* Improved error message context when invalid user data is passed to a pipeline without being wrapped in a call to constant(...)

* Fix typo in old error message

* Fix lite pipeline exports

* api-review files

* Fixing tests and improving test coverage

* PR feedback

* Updating replaceWith to take an Expr

* yarn format

Author: MarkDuckworth

common/api-review/firestore-lite-pipelines.api.md

@@ -6,7 +6,7 @@
   "module": "../../dist/lite/pipelines.browser.esm2017.js",
   "browser": "../../dist/lite/pipelines.browser.esm2017.js",
   "react-native": "../../dist/lite/pipelines.rn.esm2017.js",
-  "typings": "../../dist/lite/pipelines.d.ts",
+  "typings": "./pipelines.d.ts",
   "private": true,
   "engines": {
     "node": ">=18.0.0"

common/api-review/firestore-pipelines.api.md

@@ -19,7 +19,7 @@ import { PipelineSource, Pipeline } from '../../dist/lite/pipelines';
 // Augument the Firestore class with the pipeline() method.
 // This is stripped from dist/lite/pipelines.d.ts during the build
 // so it needs to be re-added here.
-declare module '@firebase/firestore' {
+declare module '@firebase/firestore/lite' {
   interface Firestore {
     pipeline(): PipelineSource<Pipeline>;
   }

common/api-review/firestore.api.md

@@ -50,11 +50,14 @@ export type {
 
 export { PipelineSource } from '../../src/lite-api/pipeline-source';
 
-export { PipelineResult } from '../../src/lite-api/pipeline-result';
+export {
+  PipelineResult,
+  PipelineSnapshot
+} from '../../src/lite-api/pipeline-result';
 
 export { Pipeline } from '../../src/lite-api/pipeline';
 
-export { pipeline, execute } from '../../src/lite-api/pipeline_impl';
+export { execute } from '../../src/lite-api/pipeline_impl';
 
 export {
   Stage,
@@ -76,14 +79,44 @@ export {
 } from '../../src/lite-api/stage';
 
 export {
+  Expr,
+  field,
+  and,
+  array,
+  arrayOffset,
+  constant,
   add,
   subtract,
   multiply,
+  avg,
+  bitAnd,
+  substr,
+  constantVector,
+  bitLeftShift,
+  bitNot,
+  count,
+  mapMerge,
+  mapRemove,
+  bitOr,
+  ifError,
+  isAbsent,
+  isError,
+  or,
+  rand,
+  bitRightShift,
+  bitXor,
   divide,
+  isNotNan,
+  map,
+  isNotNull,
+  isNull,
   mod,
+  documentId,
   eq,
   neq,
   lt,
+  countIf,
+  currentContext,
   lte,
   gt,
   gte,
@@ -132,79 +165,16 @@ export {
   timestampToUnixSeconds,
   timestampAdd,
   timestampSub,
-  genericFunction,
   ascending,
   descending,
   ExprWithAlias,
   Field,
-  Fields,
   Constant,
-  FirestoreFunction,
-  Add,
-  Subtract,
-  Multiply,
-  Divide,
-  Mod,
-  Eq,
-  Neq,
-  Lt,
-  Lte,
-  Gt,
-  Gte,
-  ArrayConcat,
-  ArrayReverse,
-  ArrayContains,
-  ArrayContainsAll,
-  ArrayContainsAny,
-  ArrayLength,
-  ArrayElement,
-  EqAny,
-  IsNan,
-  Exists,
-  Not,
-  And,
-  Or,
-  Xor,
-  Cond,
-  LogicalMaximum,
-  LogicalMinimum,
-  Reverse,
-  ReplaceFirst,
-  ReplaceAll,
-  CharLength,
-  ByteLength,
-  Like,
-  RegexContains,
-  RegexMatch,
-  StrContains,
-  StartsWith,
-  EndsWith,
-  ToLower,
-  ToUpper,
-  Trim,
-  StrConcat,
-  MapGet,
-  Count,
-  Sum,
-  Avg,
-  Minimum,
-  Maximum,
-  CosineDistance,
-  DotProduct,
-  EuclideanDistance,
-  VectorLength,
-  UnixMicrosToTimestamp,
-  TimestampToUnixMicros,
-  UnixMillisToTimestamp,
-  TimestampToUnixMillis,
-  UnixSecondsToTimestamp,
-  TimestampToUnixSeconds,
-  TimestampAdd,
-  TimestampSub,
+  FunctionExpr,
   Ordering,
   ExprType,
-  AccumulatorTarget,
+  AggregateWithAlias,
   Selectable,
-  FilterCondition,
-  Accumulator
+  BooleanExpr,
+  AggregateFunction
 } from '../../src/lite-api/expressions';

packages/firestore/lite/pipelines/package.json

@@ -48,12 +48,11 @@
     "test:minified": "(cd ../../integration/firestore ; yarn test)",
     "trusted-type-check": "tsec -p tsconfig.json --noEmit",
     "api-report:main": "TS_NODE_COMPILER_OPTIONS='{\"module\":\"commonjs\"}' ts-node ../../repo-scripts/prune-dts/extract-public-api.ts --package firestore --packageRoot . --typescriptDts ./dist/firestore/src/index.d.ts --rollupDts ./dist/private.d.ts --untrimmedRollupDts ./dist/internal.d.ts --publicDts ./dist/index.d.ts",
-    "api-report:all-packages": "TS_NODE_COMPILER_OPTIONS='{\"module\":\"commonjs\"}' ts-node ../../repo-scripts/prune-dts/extract-public-api.ts --package firestore --packageRoot . --typescriptDts ./dist/firestore/src/all_packages.d.ts --rollupDts ./dist/private.all_packages.d.ts --untrimmedRollupDts ./dist/internal.all_packages.d.ts --publicDts ./dist/all_packages.d.ts",
     "api-report:pipelines": "TS_NODE_COMPILER_OPTIONS='{\"module\":\"commonjs\"}' ts-node ../../repo-scripts/prune-dts/extract-public-api.ts --package firestore-pipelines --packageRoot . --typescriptDts ./dist/firestore/pipelines/pipelines.d.ts --rollupDts ./dist/private.pipelines.d.ts --untrimmedRollupDts ./dist/internal.pipelines.d.ts --publicDts ./dist/pipelines.d.ts --otherExportsPublicDtsFiles ./dist/index.d.ts",
     "api-report:lite": "TS_NODE_COMPILER_OPTIONS='{\"module\":\"commonjs\"}' ts-node ../../repo-scripts/prune-dts/extract-public-api.ts --package firestore-lite --packageRoot . --typescriptDts ./dist/firestore/lite/index.d.ts --rollupDts ./dist/lite/private.d.ts --untrimmedRollupDts ./dist/lite/internal.d.ts --publicDts ./dist/lite/index.d.ts",
-    "api-report:lite:pipelines": "TS_NODE_COMPILER_OPTIONS='{\"module\":\"commonjs\"}' ts-node ../../repo-scripts/prune-dts/extract-public-api.ts --package firestore-lite-pipelines --packageRoot . --typescriptDts ./dist/firestore/lite/pipelines/pipelines.d.ts --rollupDts ./dist/lite/private.pipelines.d.ts --untrimmedRollupDts ./dist/lite/internal.pipelines.d.ts --publicDts ./dist/lite/pipelines.d.ts --otherExportsPublicDtsFiles ./dist/index.d.ts",
+    "api-report:lite:pipelines": "TS_NODE_COMPILER_OPTIONS='{\"module\":\"commonjs\"}' ts-node ../../repo-scripts/prune-dts/extract-public-api.ts --package firestore-lite-pipelines --packageRoot . --typescriptDts ./dist/firestore/lite/pipelines/pipelines.d.ts --rollupDts ./dist/lite/private.pipelines.d.ts --untrimmedRollupDts ./dist/lite/internal.pipelines.d.ts --publicDts ./dist/lite/pipelines.d.ts --otherExportsPublicDtsFiles ./dist/lite/index.d.ts",
     "api-report:api-json": "rm -rf temp && api-extractor run --local --verbose",
-    "api-report": "run-s --npm-path npm api-report:main api-report:pipelines api-report:all-packages api-report:lite api-report:lite:pipelines && yarn api-report:api-json",
+    "api-report": "run-s --npm-path npm api-report:main api-report:pipelines api-report:lite api-report:lite:pipelines && yarn api-report:api-json",
     "doc": "api-documenter markdown --input temp --output docs",
     "typings:public": "node ../../scripts/build/use_typings.js ./dist/all-packages.d.ts"
   },

packages/firestore/lite/pipelines/pipelines.d.ts

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import {PipelineSource, Pipeline} from "../dist/pipelines";
+import { PipelineSource, Pipeline } from '../dist/pipelines';
 
 // Augument the Firestore and Query classes with the pipeline() method.
 // This is stripped from dist/lite/pipelines.d.ts during the build
@@ -23,9 +23,6 @@ declare module '@firebase/firestore' {
   interface Firestore {
     pipeline(): PipelineSource<Pipeline>;
   }
-  interface Query {
-    pipeline(): Pipeline;
-  }
 }
 
-export * from "../dist/pipelines";
+export * from '../dist/pipelines';

packages/firestore/lite/pipelines/pipelines.ts

@@ -224,8 +224,7 @@ export { isBase64Available as _isBase64Available } from './platform/base64';
 export { DatabaseId as _DatabaseId } from './core/database_info';
 export {
   _internalQueryToProtoQueryTarget,
-  _internalAggregationQueryToProtoRunAggregationQueryRequest,
-  _internalPipelineToExecutePipelineRequestProto
+  _internalAggregationQueryToProtoRunAggregationQueryRequest
 } from './remote/internal_serializer';
 export {
   cast as _cast,

packages/firestore/package.json

@@ -16,8 +16,53 @@
  */
 
 import { DatabaseId } from '../core/database_info';
+import { UserDataSource } from '../lite-api/user_data_reader';
+import { DocumentKey } from '../model/document_key';
+import { FieldTransform } from '../model/mutation';
+import { FieldPath as InternalFieldPath } from '../model/path';
+import { JsonProtoSerializer } from '../remote/serializer';
+import { FirestoreError } from '../util/error';
+
+/** Contains the settings that are mutated as we parse user data. */
+export interface ContextSettings {
+  /** Indicates what kind of API method this data came from. */
+  readonly dataSource: UserDataSource;
+  /** The name of the method the user called to create the ParseContext. */
+  readonly methodName: string;
+  /** The document the user is attempting to modify, if that applies. */
+  readonly targetDoc?: DocumentKey;
+  /**
+   * A path within the object being parsed. This could be an empty path (in
+   * which case the context represents the root of the data being parsed), or a
+   * nonempty path (indicating the context represents a nested location within
+   * the data).
+   */
+  readonly path?: InternalFieldPath;
+  /**
+   * Whether or not this context corresponds to an element of an array.
+   * If not set, elements are treated as if they were outside of arrays.
+   */
+  readonly arrayElement?: boolean;
+  /**
+   * Whether or not a converter was specified in this context. If true, error
+   * messages will reference the converter when invalid data is provided.
+   */
+  readonly hasConverter?: boolean;
+}
 
 export interface ParseContext {
+  readonly settings: ContextSettings;
   readonly databaseId: DatabaseId;
+  readonly serializer: JsonProtoSerializer;
   readonly ignoreUndefinedProperties: boolean;
+  fieldTransforms: FieldTransform[];
+  fieldMask: InternalFieldPath[];
+  get path(): InternalFieldPath | undefined;
+  get dataSource(): UserDataSource;
+  contextWith(configuration: Partial<ContextSettings>): ParseContext;
+  childContextForField(field: string): ParseContext;
+  childContextForFieldPath(field: InternalFieldPath): ParseContext;
+  childContextForArray(index: number): ParseContext;
+  createError(reason: string): FirestoreError;
+  contains(fieldPath: InternalFieldPath): boolean;
 }

packages/firestore/pipelines/pipelines.d.ts

@@ -15,16 +15,12 @@
  * limitations under the License.
  */
 
-import { firestoreClientExecutePipeline } from '../core/firestore_client';
 import { Pipeline as LitePipeline } from '../lite-api/pipeline';
-import { PipelineResult } from '../lite-api/pipeline-result';
-import { DocumentReference } from '../lite-api/reference';
 import { Stage } from '../lite-api/stage';
 import { UserDataReader } from '../lite-api/user_data_reader';
 import { AbstractUserDataWriter } from '../lite-api/user_data_writer';
-import { cast } from '../util/input_validation';
 
-import { ensureFirestoreConfigured, Firestore } from './database';
+import { Firestore } from './database';
 
 export class Pipeline extends LitePipeline {
   /**
@@ -41,66 +37,8 @@ export class Pipeline extends LitePipeline {
     db: Firestore,
     userDataReader: UserDataReader,
     userDataWriter: AbstractUserDataWriter,
-    stages: Stage[],
-    converter: unknown = {}
+    stages: Stage[]
   ): Pipeline {
     return new Pipeline(db, userDataReader, userDataWriter, stages);
   }
-
-  /**
-   * Executes this pipeline and returns a Promise to represent the asynchronous operation.
-   *
-   * <p>The returned Promise can be used to track the progress of the pipeline execution
-   * and retrieve the results (or handle any errors) asynchronously.
-   *
-   * <p>The pipeline results are returned as a list of {@link PipelineResult} objects. Each {@link
-   * PipelineResult} typically represents a single key/value map that has passed through all the
-   * stages of the pipeline, however this might differ depending on the stages involved in the
-   * pipeline. For example:
-   *
-   * <ul>
-   *   <li>If there are no stages or only transformation stages, each {@link PipelineResult}
-   *       represents a single document.</li>
-   *   <li>If there is an aggregation, only a single {@link PipelineResult} is returned,
-   *       representing the aggregated results over the entire dataset .</li>
-   *   <li>If there is an aggregation stage with grouping, each {@link PipelineResult} represents a
-   *       distinct group and its associated aggregated values.</li>
-   * </ul>
-   *
-   * <p>Example:
-   *
-   * ```typescript
-   * const futureResults = await firestore.pipeline().collection("books")
-   *     .where(gt(Field.of("rating"), 4.5))
-   *     .select("title", "author", "rating")
-   *     .execute();
-   * ```
-   *
-   * @return A Promise representing the asynchronous pipeline execution.
-   */
-  execute(): Promise<PipelineResult[]> {
-    const firestore = cast(this._db, Firestore);
-    const client = ensureFirestoreConfigured(firestore);
-    return firestoreClientExecutePipeline(client, this).then(result => {
-      const docs = result
-        // Currently ignore any response from ExecutePipeline that does
-        // not contain any document data in the `fields` property.
-        .filter(element => !!element.fields)
-        .map(
-          element =>
-            new PipelineResult(
-              this._userDataWriter,
-              element.key?.path
-                ? new DocumentReference(firestore, null, element.key)
-                : undefined,
-              element.fields,
-              element.executionTime?.toTimestamp(),
-              element.createTime?.toTimestamp(),
-              element.updateTime?.toTimestamp()
-            )
-        );
-
-      return docs;
-    });
-  }
 }