{ "source": "pyspark_repo", "spark_version": "18", "branch": "v3.5.0", "functions": [ { "name": "abs", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes the absolute value." }, { "name": "acos", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes inverse cosine of the input column." }, { "name": "acosh", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes inverse hyperbolic cosine of the input column." }, { "name": "add_months", "args": [ { "name": "start", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "months", "default": null, "annotation": "Union['ColumnOrName', int]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the date that is `months` months after `start`. If `months` is a negative value" }, { "name": "aes_decrypt", "args": [ { "name": "input", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "key", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "mode", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "padding", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "aad", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a decrypted value of `input` using AES in `mode` with `padding`. Key lengths of 16," }, { "name": "aes_encrypt", "args": [ { "name": "input", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "key", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "mode", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "padding", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "iv", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "aad", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns an encrypted value of `input` using AES in given `mode` with the specified `padding`." }, { "name": "aggregate", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "initialValue", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "merge", "default": null, "annotation": "Callable[[Column, Column], Column]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "finish", "default": "None", "annotation": "Optional[Callable[[Column], Column]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Applies a binary operator to an initial state and all elements in the array," }, { "name": "any_value", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "ignoreNulls", "default": "None", "annotation": "Optional[Union[bool, Column]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns some value of `col` for a group of rows." }, { "name": "approxCountDistinct", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "rsd", "default": "None", "annotation": "Optional[float]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": ".. versionadded:: 1.3.0" }, { "name": "approx_count_distinct", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "rsd", "default": "None", "annotation": "Optional[float]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns a new :class:`~pyspark.sql.Column` for approximate distinct count" }, { "name": "approx_percentile", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "percentage", "default": null, "annotation": "Union[Column, float, List[float], Tuple[float]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "accuracy", "default": "10000", "annotation": "Union[Column, float]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the approximate `percentile` of the numeric column `col` which is the smallest value" }, { "name": "array", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "" }, { "name": "array", "args": [ { "name": "__cols", "default": null, "annotation": "Union[List['ColumnOrName_'], Tuple['ColumnOrName_', ...]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "" }, { "name": "array", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Creates a new array column." }, { "name": "array_agg", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns a list of objects with duplicates." }, { "name": "array_append", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "value", "default": null, "annotation": "Any", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: returns an array of the elements in col1 along" }, { "name": "array_compact", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: removes null values from the array." }, { "name": "array_contains", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "value", "default": null, "annotation": "Any", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: returns null if the array is null, true if the array contains the" }, { "name": "array_distinct", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: removes duplicate values from the array." }, { "name": "array_except", "args": [ { "name": "col1", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col2", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: returns an array of the elements in col1 but not in col2," }, { "name": "array_insert", "args": [ { "name": "arr", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "pos", "default": null, "annotation": "Union['ColumnOrName', int]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "value", "default": null, "annotation": "Any", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: adds an item into a given array at a specified array index." }, { "name": "array_intersect", "args": [ { "name": "col1", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col2", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: returns an array of the elements in the intersection of col1 and col2," }, { "name": "array_join", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "delimiter", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "null_replacement", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Concatenates the elements of `column` using the `delimiter`. Null values are replaced with" }, { "name": "array_max", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: returns the maximum value of the array." }, { "name": "array_min", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: returns the minimum value of the array." }, { "name": "array_position", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "value", "default": null, "annotation": "Any", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: Locates the position of the first occurrence of the given value" }, { "name": "array_prepend", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "value", "default": null, "annotation": "Any", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: Returns an array containing element as" }, { "name": "array_remove", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "element", "default": null, "annotation": "Any", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: Remove all elements that equal to element from the given array." }, { "name": "array_repeat", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "count", "default": null, "annotation": "Union['ColumnOrName', int]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: creates an array containing a column repeated count times." }, { "name": "array_size", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the total number of elements in the array. The function returns null for null input." }, { "name": "array_sort", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "comparator", "default": "None", "annotation": "Optional[Callable[[Column, Column], Column]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: sorts the input array in ascending order. The elements of the input array" }, { "name": "array_union", "args": [ { "name": "col1", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col2", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: returns an array of the elements in the union of col1 and col2," }, { "name": "arrays_overlap", "args": [ { "name": "a1", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "a2", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: returns true if the arrays contain any common non-null element; if not," }, { "name": "arrays_zip", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Collection function: Returns a merged array of structs in which the N-th struct contains all" }, { "name": "asc", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a sort expression based on the ascending order of the given column name." }, { "name": "asc_nulls_first", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a sort expression based on the ascending order of the given" }, { "name": "asc_nulls_last", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a sort expression based on the ascending order of the given" }, { "name": "ascii", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes the numeric value of the first character of the string column." }, { "name": "asin", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes inverse sine of the input column." }, { "name": "asinh", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes inverse hyperbolic sine of the input column." }, { "name": "assert_true", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "errMsg", "default": "None", "annotation": "Optional[Union[Column, str]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns `null` if the input column is `true`; throws an exception" }, { "name": "atan", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Compute inverse tangent of the input column." }, { "name": "atan2", "args": [ { "name": "col1", "default": null, "annotation": "Union['ColumnOrName', float]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col2", "default": null, "annotation": "Union['ColumnOrName', float]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": ".. versionadded:: 1.4.0" }, { "name": "atanh", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes inverse hyperbolic tangent of the input column." }, { "name": "avg", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the average of the values in a group." }, { "name": "base64", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes the BASE64 encoding of a binary column and returns it as a string column." }, { "name": "bin", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the string representation of the binary value of the given column." }, { "name": "bit_and", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the bitwise AND of all non-null input values, or null if none." }, { "name": "bit_count", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the number of bits that are set in the argument expr as an unsigned 64-bit integer," }, { "name": "bit_get", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "pos", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the value of the bit (0 or 1) at the specified position." }, { "name": "bit_length", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Calculates the bit length for the specified string column." }, { "name": "bit_or", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the bitwise OR of all non-null input values, or null if none." }, { "name": "bit_xor", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the bitwise XOR of all non-null input values, or null if none." }, { "name": "bitmap_bit_position", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the bit position for the given input column." }, { "name": "bitmap_bucket_number", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the bucket number for the given input column." }, { "name": "bitmap_construct_agg", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a bitmap with the positions of the bits set from all the values from the input column." }, { "name": "bitmap_count", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the number of set bits in the input bitmap." }, { "name": "bitmap_or_agg", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a bitmap that is the bitwise OR of all of the bitmaps from the input column." }, { "name": "bitwiseNOT", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes bitwise not." }, { "name": "bitwise_not", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes bitwise not." }, { "name": "bool_and", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns true if all values of `col` are true." }, { "name": "bool_or", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns true if at least one value of `col` is true." }, { "name": "broadcast", "args": [ { "name": "df", "default": null, "annotation": "DataFrame", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Marks a DataFrame as small enough for use in broadcast joins." }, { "name": "bround", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "scale", "default": "0", "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Round the given value to `scale` decimal places using HALF_EVEN rounding mode if `scale` >= 0" }, { "name": "btrim", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "trim", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Remove the leading and trailing `trim` characters from `str`." }, { "name": "bucket", "args": [ { "name": "numBuckets", "default": null, "annotation": "Union[Column, int]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Partition transform function: A transform for any type that partitions" }, { "name": "call_function", "args": [ { "name": "funcName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Call a SQL function." }, { "name": "call_udf", "args": [ { "name": "udfName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Call an user-defined function." }, { "name": "cardinality", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: returns the length of the array or map stored in the column." }, { "name": "cbrt", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes the cube-root of the given value." }, { "name": "ceil", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes the ceiling of the given value." }, { "name": "ceiling", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes the ceiling of the given value." }, { "name": "char", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the ASCII character having the binary equivalent to `col`. If col is larger than 256 the" }, { "name": "char_length", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the character length of string data or number of bytes of binary data." }, { "name": "character_length", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the character length of string data or number of bytes of binary data." }, { "name": "coalesce", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Returns the first column that is not null." }, { "name": "col", "args": [ { "name": "col", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a :class:`~pyspark.sql.Column` based on the given column name." }, { "name": "collect_list", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns a list of objects with duplicates." }, { "name": "collect_set", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns a set of objects with duplicate elements eliminated." }, { "name": "concat", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Concatenates multiple input columns together into a single column." }, { "name": "concat_ws", "args": [ { "name": "sep", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Concatenates multiple input string columns together into a single string column," }, { "name": "contains", "args": [ { "name": "left", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "right", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a boolean. The value is True if right is found inside left." }, { "name": "conv", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "fromBase", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "toBase", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Convert a number in a string column from one base to another." }, { "name": "convert_timezone", "args": [ { "name": "sourceTz", "default": null, "annotation": "Optional[Column]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "targetTz", "default": null, "annotation": "Column", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "sourceTs", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Converts the timestamp without time zone `sourceTs`" }, { "name": "corr", "args": [ { "name": "col1", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col2", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a new :class:`~pyspark.sql.Column` for the Pearson Correlation Coefficient for" }, { "name": "cos", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes cosine of the input column." }, { "name": "cosh", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes hyperbolic cosine of the input column." }, { "name": "cot", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes cotangent of the input column." }, { "name": "count", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the number of items in a group." }, { "name": "countDistinct", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Returns a new :class:`~pyspark.sql.Column` for distinct count of ``col`` or ``cols``." }, { "name": "count_distinct", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Returns a new :class:`Column` for distinct count of ``col`` or ``cols``." }, { "name": "count_if", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the number of `TRUE` values for the `col`." }, { "name": "count_min_sketch", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "eps", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "confidence", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "seed", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a count-min sketch of a column with the given esp, confidence and seed." }, { "name": "covar_pop", "args": [ { "name": "col1", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col2", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a new :class:`~pyspark.sql.Column` for the population covariance of ``col1`` and" }, { "name": "covar_samp", "args": [ { "name": "col1", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col2", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a new :class:`~pyspark.sql.Column` for the sample covariance of ``col1`` and" }, { "name": "crc32", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Calculates the cyclic redundancy check value (CRC32) of a binary column and" }, { "name": "create_map", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "" }, { "name": "create_map", "args": [ { "name": "__cols", "default": null, "annotation": "Union[List['ColumnOrName_'], Tuple['ColumnOrName_', ...]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "" }, { "name": "create_map", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Creates a new map column." }, { "name": "csc", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes cosecant of the input column." }, { "name": "cume_dist", "args": [], "doc_preview": "Window function: returns the cumulative distribution of values within a window partition," }, { "name": "curdate", "args": [], "doc_preview": "Returns the current date at the start of query evaluation as a :class:`DateType` column." }, { "name": "current_catalog", "args": [], "doc_preview": "Returns the current catalog." }, { "name": "current_database", "args": [], "doc_preview": "Returns the current database." }, { "name": "current_date", "args": [], "doc_preview": "Returns the current date at the start of query evaluation as a :class:`DateType` column." }, { "name": "current_schema", "args": [], "doc_preview": "Returns the current database." }, { "name": "current_timestamp", "args": [], "doc_preview": "Returns the current timestamp at the start of query evaluation as a :class:`TimestampType`" }, { "name": "current_timezone", "args": [], "doc_preview": "Returns the current session local timezone." }, { "name": "current_user", "args": [], "doc_preview": "Returns the current database." }, { "name": "date_add", "args": [ { "name": "start", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "days", "default": null, "annotation": "Union['ColumnOrName', int]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the date that is `days` days after `start`. If `days` is a negative value" }, { "name": "date_diff", "args": [ { "name": "end", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "start", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the number of days from `start` to `end`." }, { "name": "date_format", "args": [ { "name": "date", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "format", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Converts a date/timestamp/string to a value of string in the format specified by the date" }, { "name": "date_from_unix_date", "args": [ { "name": "days", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Create date from the number of `days` since 1970-01-01." }, { "name": "date_part", "args": [ { "name": "field", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "source", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Extracts a part of the date/timestamp or interval source." }, { "name": "date_sub", "args": [ { "name": "start", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "days", "default": null, "annotation": "Union['ColumnOrName', int]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the date that is `days` days before `start`. If `days` is a negative value" }, { "name": "date_trunc", "args": [ { "name": "format", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "timestamp", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns timestamp truncated to the unit specified by the format." }, { "name": "dateadd", "args": [ { "name": "start", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "days", "default": null, "annotation": "Union['ColumnOrName', int]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the date that is `days` days after `start`. If `days` is a negative value" }, { "name": "datediff", "args": [ { "name": "end", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "start", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the number of days from `start` to `end`." }, { "name": "datepart", "args": [ { "name": "field", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "source", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Extracts a part of the date/timestamp or interval source." }, { "name": "day", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Extract the day of the month of a given date/timestamp as integer." }, { "name": "dayofmonth", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Extract the day of the month of a given date/timestamp as integer." }, { "name": "dayofweek", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Extract the day of the week of a given date/timestamp as integer." }, { "name": "dayofyear", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Extract the day of the year of a given date/timestamp as integer." }, { "name": "days", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Partition transform function: A transform for timestamps and dates" }, { "name": "decode", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "charset", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes the first argument into a string from a binary using the provided character set" }, { "name": "degrees", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Converts an angle measured in radians to an approximately equivalent angle" }, { "name": "dense_rank", "args": [], "doc_preview": "Window function: returns the rank of rows within a window partition, without any gaps." }, { "name": "desc", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a sort expression based on the descending order of the given column name." }, { "name": "desc_nulls_first", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a sort expression based on the descending order of the given" }, { "name": "desc_nulls_last", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a sort expression based on the descending order of the given" }, { "name": "e", "args": [], "doc_preview": "Returns Euler's number." }, { "name": "element_at", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "extraction", "default": null, "annotation": "Any", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: Returns element of array at given index in `extraction` if col is array." }, { "name": "elt", "args": [ { "name": "inputs", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Returns the `n`-th input, e.g., returns `input2` when `n` is 2." }, { "name": "encode", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "charset", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes the first argument into a binary from a string using the provided character set" }, { "name": "endswith", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "suffix", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a boolean. The value is True if str ends with suffix." }, { "name": "equal_null", "args": [ { "name": "col1", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col2", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns same result as the EQUAL(=) operator for non-null operands," }, { "name": "every", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns true if all values of `col` are true." }, { "name": "exists", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "f", "default": null, "annotation": "Callable[[Column], Column]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns whether a predicate holds for one or more elements in the array." }, { "name": "exp", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes the exponential of the given value." }, { "name": "explode", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a new row for each element in the given array or map." }, { "name": "explode_outer", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a new row for each element in the given array or map." }, { "name": "expm1", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes the exponential of the given value minus one." }, { "name": "expr", "args": [ { "name": "str", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Parses the expression string into the column that it represents" }, { "name": "extract", "args": [ { "name": "field", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "source", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Extracts a part of the date/timestamp or interval source." }, { "name": "factorial", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes the factorial of the given value." }, { "name": "filter", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "f", "default": null, "annotation": "Callable[[Column], Column]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "" }, { "name": "filter", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "f", "default": null, "annotation": "Callable[[Column, Column], Column]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "" }, { "name": "filter", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "f", "default": null, "annotation": "Union[Callable[[Column], Column], Callable[[Column, Column], Column]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns an array of elements for which a predicate holds in a given array." }, { "name": "find_in_set", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "str_array", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the index (1-based) of the given string (`str`) in the comma-delimited" }, { "name": "first", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "ignorenulls", "default": "False", "annotation": "bool", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the first value in a group." }, { "name": "first_value", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "ignoreNulls", "default": "None", "annotation": "Optional[Union[bool, Column]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the first value of `col` for a group of rows. It will return the first non-null" }, { "name": "flatten", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: creates a single array from an array of arrays." }, { "name": "floor", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes the floor of the given value." }, { "name": "forall", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "f", "default": null, "annotation": "Callable[[Column], Column]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns whether a predicate holds for every element in the array." }, { "name": "format_number", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "d", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Formats the number X to a format like '#,--#,--#.--', rounded to d decimal places" }, { "name": "format_string", "args": [ { "name": "format", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Formats the arguments in printf-style and returns the result as a string column." }, { "name": "from_csv", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "schema", "default": null, "annotation": "Union[Column, str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "options", "default": "None", "annotation": "Optional[Dict[str, str]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Parses a column containing a CSV string to a row with the specified schema." }, { "name": "from_json", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "schema", "default": null, "annotation": "Union[ArrayType, StructType, Column, str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "options", "default": "None", "annotation": "Optional[Dict[str, str]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Parses a column containing a JSON string into a :class:`MapType` with :class:`StringType`" }, { "name": "from_unixtime", "args": [ { "name": "timestamp", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "format", "default": "'yyyy-MM-dd HH:mm:ss'", "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to a string" }, { "name": "from_utc_timestamp", "args": [ { "name": "timestamp", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "tz", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "This is a common function for databases supporting TIMESTAMP WITHOUT TIMEZONE. This function" }, { "name": "get", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "index", "default": null, "annotation": "Union['ColumnOrName', int]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: Returns element of array at given (0-based) index." }, { "name": "get_json_object", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "path", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Extracts json object from a json string based on json `path` specified, and returns json string" }, { "name": "getbit", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "pos", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the value of the bit (0 or 1) at the specified position." }, { "name": "greatest", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Returns the greatest value of the list of column names, skipping null values." }, { "name": "grouping", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated" }, { "name": "grouping_id", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Aggregate function: returns the level of grouping, equals to" }, { "name": "hash", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Calculates the hash code of given columns, and returns the result as an int column." }, { "name": "hex", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes hex value of the given column, which could be :class:`pyspark.sql.types.StringType`," }, { "name": "histogram_numeric", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "nBins", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes a histogram on numeric 'col' using nb bins." }, { "name": "hll_sketch_agg", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "lgConfigK", "default": "None", "annotation": "Optional[Union[int, Column]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the updatable binary representation of the Datasketches" }, { "name": "hll_sketch_estimate", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the estimated number of unique values given the binary representation" }, { "name": "hll_union", "args": [ { "name": "col1", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col2", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "allowDifferentLgConfigK", "default": "None", "annotation": "Optional[bool]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Merges two binary representations of Datasketches HllSketch objects, using a" }, { "name": "hll_union_agg", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "allowDifferentLgConfigK", "default": "None", "annotation": "Optional[Union[bool, Column]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the updatable binary representation of the Datasketches" }, { "name": "hour", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Extract the hours of a given timestamp as integer." }, { "name": "hours", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Partition transform function: A transform for timestamps" }, { "name": "hypot", "args": [ { "name": "col1", "default": null, "annotation": "Union['ColumnOrName', float]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col2", "default": null, "annotation": "Union['ColumnOrName', float]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes ``sqrt(a^2 + b^2)`` without intermediate overflow or underflow." }, { "name": "ifnull", "args": [ { "name": "col1", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col2", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns `col2` if `col1` is null, or `col1` otherwise." }, { "name": "ilike", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "pattern", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "escapeChar", "default": "None", "annotation": "Optional['Column']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns true if str matches `pattern` with `escape` case-insensitively," }, { "name": "initcap", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Translate the first letter of each word to upper case in the sentence." }, { "name": "inline", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Explodes an array of structs into a table." }, { "name": "inline_outer", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Explodes an array of structs into a table." }, { "name": "input_file_block_length", "args": [], "doc_preview": "Returns the length of the block being read, or -1 if not available." }, { "name": "input_file_block_start", "args": [], "doc_preview": "Returns the start offset of the block being read, or -1 if not available." }, { "name": "input_file_name", "args": [], "doc_preview": "Creates a string column for the file name of the current Spark task." }, { "name": "instr", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "substr", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Locate the position of the first occurrence of substr column in the given string." }, { "name": "isnan", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "An expression that returns true if the column is NaN." }, { "name": "isnotnull", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns true if `col` is not null, or false otherwise." }, { "name": "isnull", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "An expression that returns true if the column is null." }, { "name": "java_method", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Calls a method with reflection." }, { "name": "json_array_length", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the number of elements in the outermost JSON array. `NULL` is returned in case of" }, { "name": "json_object_keys", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns all the keys of the outermost JSON object as an array. If a valid JSON object is" }, { "name": "json_tuple", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "fields", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Creates a new row for a json column according to the given field names." }, { "name": "kurtosis", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the kurtosis of the values in a group." }, { "name": "lag", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "offset", "default": "1", "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "default", "default": "None", "annotation": "Optional[Any]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Window function: returns the value that is `offset` rows before the current row, and" }, { "name": "last", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "ignorenulls", "default": "False", "annotation": "bool", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the last value in a group." }, { "name": "last_day", "args": [ { "name": "date", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the last day of the month which the given date belongs to." }, { "name": "last_value", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "ignoreNulls", "default": "None", "annotation": "Optional[Union[bool, Column]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the last value of `col` for a group of rows. It will return the last non-null" }, { "name": "lcase", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns `str` with all characters changed to lowercase." }, { "name": "lead", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "offset", "default": "1", "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "default", "default": "None", "annotation": "Optional[Any]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Window function: returns the value that is `offset` rows after the current row, and" }, { "name": "least", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Returns the least value of the list of column names, skipping null values." }, { "name": "left", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "len", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the leftmost `len`(`len` can be string type) characters from the string `str`," }, { "name": "length", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes the character length of string data or number of bytes of binary data." }, { "name": "levenshtein", "args": [ { "name": "left", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "right", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "threshold", "default": "None", "annotation": "Optional[int]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes the Levenshtein distance of the two given strings." }, { "name": "like", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "pattern", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "escapeChar", "default": "None", "annotation": "Optional['Column']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns true if str matches `pattern` with `escape`," }, { "name": "lit", "args": [ { "name": "col", "default": null, "annotation": "Any", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Creates a :class:`~pyspark.sql.Column` of literal value." }, { "name": "ln", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the natural logarithm of the argument." }, { "name": "localtimestamp", "args": [], "doc_preview": "Returns the current timestamp without time zone at the start of query evaluation" }, { "name": "locate", "args": [ { "name": "substr", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "pos", "default": "1", "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Locate the position of the first occurrence of substr in a string column, after position pos." }, { "name": "log", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes the natural logarithm of the given value." }, { "name": "log", "args": [ { "name": "arg1", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "" }, { "name": "log", "args": [ { "name": "arg1", "default": null, "annotation": "float", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "arg2", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "" }, { "name": "log", "args": [ { "name": "arg1", "default": null, "annotation": "Union['ColumnOrName', float]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "arg2", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the first argument-based logarithm of the second argument." }, { "name": "log10", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes the logarithm of the given value in Base 10." }, { "name": "log1p", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes the natural logarithm of the \"given value plus one\"." }, { "name": "log2", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the base-2 logarithm of the argument." }, { "name": "lower", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Converts a string expression to lower case." }, { "name": "lpad", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "len", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "pad", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Left-pad the string column to width `len` with `pad`." }, { "name": "ltrim", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Trim the spaces from left end for the specified string value." }, { "name": "make_date", "args": [ { "name": "year", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "month", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "day", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a column with a date built from the year, month and day columns." }, { "name": "make_dt_interval", "args": [ { "name": "days", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "hours", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "mins", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "secs", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Make DayTimeIntervalType duration from days, hours, mins and secs." }, { "name": "make_interval", "args": [ { "name": "years", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "months", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "weeks", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "days", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "hours", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "mins", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "secs", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Make interval from years, months, weeks, days, hours, mins and secs." }, { "name": "make_timestamp", "args": [ { "name": "years", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "months", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "days", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "hours", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "mins", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "secs", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "timezone", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Create timestamp from years, months, days, hours, mins, secs and timezone fields." }, { "name": "make_timestamp_ltz", "args": [ { "name": "years", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "months", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "days", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "hours", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "mins", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "secs", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "timezone", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Create the current timestamp with local time zone from years, months, days, hours, mins," }, { "name": "make_timestamp_ntz", "args": [ { "name": "years", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "months", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "days", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "hours", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "mins", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "secs", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Create local date-time from years, months, days, hours, mins, secs fields." }, { "name": "make_ym_interval", "args": [ { "name": "years", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "months", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Make year-month interval from years, months." }, { "name": "map_concat", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "" }, { "name": "map_concat", "args": [ { "name": "__cols", "default": null, "annotation": "Union[List['ColumnOrName_'], Tuple['ColumnOrName_', ...]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "" }, { "name": "map_concat", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Returns the union of all the given maps." }, { "name": "map_contains_key", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "value", "default": null, "annotation": "Any", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns true if the map contains the key." }, { "name": "map_entries", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: Returns an unordered array of all entries in the given map." }, { "name": "map_filter", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "f", "default": null, "annotation": "Callable[[Column, Column], Column]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a map whose key-value pairs satisfy a predicate." }, { "name": "map_from_arrays", "args": [ { "name": "col1", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col2", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Creates a new map from two arrays." }, { "name": "map_from_entries", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: Converts an array of entries (key value struct types) to a map" }, { "name": "map_keys", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: Returns an unordered array containing the keys of the map." }, { "name": "map_values", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: Returns an unordered array containing the values of the map." }, { "name": "map_zip_with", "args": [ { "name": "col1", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col2", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "f", "default": null, "annotation": "Callable[[Column, Column, Column], Column]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Merge two given maps, key-wise into a single map using a function." }, { "name": "mask", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "upperChar", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "lowerChar", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "digitChar", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "otherChar", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Masks the given string value. This can be useful for creating copies of tables with sensitive" }, { "name": "max", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the maximum value of the expression in a group." }, { "name": "max_by", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "ord", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the value associated with the maximum value of ord." }, { "name": "md5", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Calculates the MD5 digest and returns the value as a 32 character hex string." }, { "name": "mean", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the average of the values in a group." }, { "name": "median", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the median of the values in a group." }, { "name": "min", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the minimum value of the expression in a group." }, { "name": "min_by", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "ord", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the value associated with the minimum value of ord." }, { "name": "minute", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Extract the minutes of a given timestamp as integer." }, { "name": "mode", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the most frequent value in a group." }, { "name": "monotonically_increasing_id", "args": [], "doc_preview": "A column that generates monotonically increasing 64-bit integers." }, { "name": "month", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Extract the month of a given date/timestamp as integer." }, { "name": "months", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Partition transform function: A transform for timestamps and dates" }, { "name": "months_between", "args": [ { "name": "date1", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "date2", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "roundOff", "default": "True", "annotation": "bool", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns number of months between dates date1 and date2." }, { "name": "named_struct", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Creates a struct with the given field names and values." }, { "name": "nanvl", "args": [ { "name": "col1", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col2", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns col1 if it is not NaN, or col2 if col1 is NaN." }, { "name": "negative", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the negative value." }, { "name": "next_day", "args": [ { "name": "date", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "dayOfWeek", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the first date which is later than the value of the date column" }, { "name": "now", "args": [], "doc_preview": "Returns the current timestamp at the start of query evaluation." }, { "name": "nth_value", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "offset", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "ignoreNulls", "default": "False", "annotation": "Optional[bool]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Window function: returns the value that is the `offset`\\th row of the window frame" }, { "name": "ntile", "args": [ { "name": "n", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Window function: returns the ntile group id (from 1 to `n` inclusive)" }, { "name": "nullif", "args": [ { "name": "col1", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col2", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns null if `col1` equals to `col2`, or `col1` otherwise." }, { "name": "nvl", "args": [ { "name": "col1", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col2", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns `col2` if `col1` is null, or `col1` otherwise." }, { "name": "nvl2", "args": [ { "name": "col1", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col2", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col3", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns `col2` if `col1` is not null, or `col3` otherwise." }, { "name": "octet_length", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Calculates the byte length for the specified string column." }, { "name": "overlay", "args": [ { "name": "src", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "replace", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "pos", "default": null, "annotation": "Union['ColumnOrName', int]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "len", "default": "-1", "annotation": "Union['ColumnOrName', int]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Overlay the specified portion of `src` with `replace`," }, { "name": "parse_url", "args": [ { "name": "url", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "partToExtract", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "key", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Extracts a part from a URL." }, { "name": "percent_rank", "args": [], "doc_preview": "Window function: returns the relative rank (i.e. percentile) of rows within a window partition." }, { "name": "percentile", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "percentage", "default": null, "annotation": "Union[Column, float, List[float], Tuple[float]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "frequency", "default": "1", "annotation": "Union[Column, int]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the exact percentile(s) of numeric column `expr` at the given percentage(s)" }, { "name": "percentile_approx", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "percentage", "default": null, "annotation": "Union[Column, float, List[float], Tuple[float]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "accuracy", "default": "10000", "annotation": "Union[Column, float]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the approximate `percentile` of the numeric column `col` which is the smallest value" }, { "name": "pi", "args": [], "doc_preview": "Returns Pi." }, { "name": "pmod", "args": [ { "name": "dividend", "default": null, "annotation": "Union['ColumnOrName', float]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "divisor", "default": null, "annotation": "Union['ColumnOrName', float]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the positive value of dividend mod divisor." }, { "name": "posexplode", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a new row for each element with position in the given array or map." }, { "name": "posexplode_outer", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a new row for each element with position in the given array or map." }, { "name": "position", "args": [ { "name": "substr", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "start", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the position of the first occurrence of `substr` in `str` after position `start`." }, { "name": "positive", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the value." }, { "name": "pow", "args": [ { "name": "col1", "default": null, "annotation": "Union['ColumnOrName', float]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col2", "default": null, "annotation": "Union['ColumnOrName', float]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the value of the first argument raised to the power of the second argument." }, { "name": "printf", "args": [ { "name": "format", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Formats the arguments in printf-style and returns the result as a string column." }, { "name": "product", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the product of the values in a group." }, { "name": "quarter", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Extract the quarter of a given date/timestamp as integer." }, { "name": "radians", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Converts an angle measured in degrees to an approximately equivalent angle" }, { "name": "raise_error", "args": [ { "name": "errMsg", "default": null, "annotation": "Union[Column, str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Throws an exception with the provided error message." }, { "name": "rand", "args": [ { "name": "seed", "default": "None", "annotation": "Optional[int]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Generates a random column with independent and identically distributed (i.i.d.) samples" }, { "name": "randn", "args": [ { "name": "seed", "default": "None", "annotation": "Optional[int]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Generates a column with independent and identically distributed (i.i.d.) samples from" }, { "name": "rank", "args": [], "doc_preview": "Window function: returns the rank of rows within a window partition." }, { "name": "reduce", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "initialValue", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "merge", "default": null, "annotation": "Callable[[Column, Column], Column]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "finish", "default": "None", "annotation": "Optional[Callable[[Column], Column]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Applies a binary operator to an initial state and all elements in the array," }, { "name": "reflect", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Calls a method with reflection." }, { "name": "regexp", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "regexp", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns true if `str` matches the Java regex `regexp`, or false otherwise." }, { "name": "regexp_count", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "regexp", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a count of the number of times that the Java regex pattern `regexp` is matched" }, { "name": "regexp_extract", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "pattern", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "idx", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Extract a specific group matched by the Java regex `regexp`, from the specified string column." }, { "name": "regexp_extract_all", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "regexp", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "idx", "default": "None", "annotation": "Optional[Union[int, Column]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Extract all strings in the `str` that match the Java regex `regexp`" }, { "name": "regexp_instr", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "regexp", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "idx", "default": "None", "annotation": "Optional[Union[int, Column]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Extract all strings in the `str` that match the Java regex `regexp`" }, { "name": "regexp_like", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "regexp", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns true if `str` matches the Java regex `regexp`, or false otherwise." }, { "name": "regexp_replace", "args": [ { "name": "string", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "pattern", "default": null, "annotation": "Union[str, Column]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "replacement", "default": null, "annotation": "Union[str, Column]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Replace all substrings of the specified string value that match regexp with replacement." }, { "name": "regexp_substr", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "regexp", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the substring that matches the Java regex `regexp` within the string `str`." }, { "name": "regr_avgx", "args": [ { "name": "y", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "x", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the average of the independent variable for non-null pairs" }, { "name": "regr_avgy", "args": [ { "name": "y", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "x", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the average of the dependent variable for non-null pairs" }, { "name": "regr_count", "args": [ { "name": "y", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "x", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the number of non-null number pairs" }, { "name": "regr_intercept", "args": [ { "name": "y", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "x", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the intercept of the univariate linear regression line" }, { "name": "regr_r2", "args": [ { "name": "y", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "x", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the coefficient of determination for non-null pairs" }, { "name": "regr_slope", "args": [ { "name": "y", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "x", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the slope of the linear regression line for non-null pairs" }, { "name": "regr_sxx", "args": [ { "name": "y", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "x", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(x) for non-null pairs" }, { "name": "regr_sxy", "args": [ { "name": "y", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "x", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns REGR_COUNT(y, x) * COVAR_POP(y, x) for non-null pairs" }, { "name": "regr_syy", "args": [ { "name": "y", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "x", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(y) for non-null pairs" }, { "name": "repeat", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "n", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Repeats a string column n times, and returns it as a new string column." }, { "name": "replace", "args": [ { "name": "src", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "search", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "replace", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Replaces all occurrences of `search` with `replace`." }, { "name": "reverse", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: returns a reversed string or an array with reverse order of elements." }, { "name": "right", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "len", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the rightmost `len`(`len` can be string type) characters from the string `str`," }, { "name": "rint", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the double value that is closest in value to the argument and" }, { "name": "rlike", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "regexp", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns true if `str` matches the Java regex `regexp`, or false otherwise." }, { "name": "round", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "scale", "default": "0", "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Round the given value to `scale` decimal places using HALF_UP rounding mode if `scale` >= 0" }, { "name": "row_number", "args": [], "doc_preview": "Window function: returns a sequential number starting at 1 within a window partition." }, { "name": "rpad", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "len", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "pad", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Right-pad the string column to width `len` with `pad`." }, { "name": "rtrim", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Trim the spaces from right end for the specified string value." }, { "name": "schema_of_csv", "args": [ { "name": "csv", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "options", "default": "None", "annotation": "Optional[Dict[str, str]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Parses a CSV string and infers its schema in DDL format." }, { "name": "schema_of_json", "args": [ { "name": "json", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "options", "default": "None", "annotation": "Optional[Dict[str, str]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Parses a JSON string and infers its schema in DDL format." }, { "name": "sec", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes secant of the input column." }, { "name": "second", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Extract the seconds of a given date as integer." }, { "name": "sentences", "args": [ { "name": "string", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "language", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "country", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Splits a string into arrays of sentences, where each sentence is an array of words." }, { "name": "sequence", "args": [ { "name": "start", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "stop", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "step", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Generate a sequence of integers from `start` to `stop`, incrementing by `step`." }, { "name": "session_window", "args": [ { "name": "timeColumn", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "gapDuration", "default": null, "annotation": "Union[Column, str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Generates session window given a timestamp specifying column." }, { "name": "sha", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a sha1 hash value as a hex string of the `col`." }, { "name": "sha1", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the hex string result of SHA-1." }, { "name": "sha2", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "numBits", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the hex string result of SHA-2 family of hash functions (SHA-224, SHA-256, SHA-384," }, { "name": "shiftLeft", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "numBits", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Shift the given value numBits left." }, { "name": "shiftRight", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "numBits", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "(Signed) shift the given value numBits right." }, { "name": "shiftRightUnsigned", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "numBits", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Unsigned shift the given value numBits right." }, { "name": "shiftleft", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "numBits", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Shift the given value numBits left." }, { "name": "shiftright", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "numBits", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "(Signed) shift the given value numBits right." }, { "name": "shiftrightunsigned", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "numBits", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Unsigned shift the given value numBits right." }, { "name": "shuffle", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: Generates a random permutation of the given array." }, { "name": "sign", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes the signum of the given value." }, { "name": "signum", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes the signum of the given value." }, { "name": "sin", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes sine of the input column." }, { "name": "sinh", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes hyperbolic sine of the input column." }, { "name": "size", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: returns the length of the array or map stored in the column." }, { "name": "skewness", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the skewness of the values in a group." }, { "name": "slice", "args": [ { "name": "x", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "start", "default": null, "annotation": "Union['ColumnOrName', int]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "length", "default": null, "annotation": "Union['ColumnOrName', int]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: returns an array containing all the elements in `x` from index `start`" }, { "name": "some", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns true if at least one value of `col` is true." }, { "name": "sort_array", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "asc", "default": "True", "annotation": "bool", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Collection function: sorts the input array in ascending or descending order according" }, { "name": "soundex", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the SoundEx encoding for a string" }, { "name": "spark_partition_id", "args": [], "doc_preview": "A column for partition ID." }, { "name": "split", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "pattern", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "limit", "default": "-1", "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Splits str around matches of the given pattern." }, { "name": "split_part", "args": [ { "name": "src", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "delimiter", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "partNum", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Splits `str` by delimiter and return requested part of the split (1-based)." }, { "name": "sqrt", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes the square root of the specified float value." }, { "name": "stack", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Separates `col1`, ..., `colk` into `n` rows. Uses column names col0, col1, etc. by default" }, { "name": "startswith", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "prefix", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a boolean. The value is True if str starts with prefix." }, { "name": "std", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: alias for stddev_samp." }, { "name": "stddev", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: alias for stddev_samp." }, { "name": "stddev_pop", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns population standard deviation of" }, { "name": "stddev_samp", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the unbiased sample standard deviation of" }, { "name": "str_to_map", "args": [ { "name": "text", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "pairDelim", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "keyValueDelim", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Creates a map after splitting the text into key/value pairs using delimiters." }, { "name": "struct", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "" }, { "name": "struct", "args": [ { "name": "__cols", "default": null, "annotation": "Union[List['ColumnOrName_'], Tuple['ColumnOrName_', ...]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "" }, { "name": "struct", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Creates a new struct column." }, { "name": "substr", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "pos", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "len", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the substring of `str` that starts at `pos` and is of length `len`," }, { "name": "substring", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "pos", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "len", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Substring starts at `pos` and is of length `len` when str is String type or" }, { "name": "substring_index", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "delim", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "count", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the substring from string str before count occurrences of the delimiter delim." }, { "name": "sum", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the sum of all values in the expression." }, { "name": "sumDistinct", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the sum of distinct values in the expression." }, { "name": "sum_distinct", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the sum of distinct values in the expression." }, { "name": "tan", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes tangent of the input column." }, { "name": "tanh", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes hyperbolic tangent of the input column." }, { "name": "timestamp_micros", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Creates timestamp from the number of microseconds since UTC epoch." }, { "name": "timestamp_millis", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Creates timestamp from the number of milliseconds since UTC epoch." }, { "name": "timestamp_seconds", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Converts the number of seconds from the Unix epoch (1970-01-01T00:00:00Z)" }, { "name": "toDegrees", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": ".. versionadded:: 1.4.0" }, { "name": "toRadians", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": ".. versionadded:: 1.4.0" }, { "name": "to_binary", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "format", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Converts the input `col` to a binary value based on the supplied `format`." }, { "name": "to_char", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "format", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Convert `col` to a string based on the `format`." }, { "name": "to_csv", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "options", "default": "None", "annotation": "Optional[Dict[str, str]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Converts a column containing a :class:`StructType` into a CSV string." }, { "name": "to_date", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "format", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Converts a :class:`~pyspark.sql.Column` into :class:`pyspark.sql.types.DateType`" }, { "name": "to_json", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "options", "default": "None", "annotation": "Optional[Dict[str, str]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Converts a column containing a :class:`StructType`, :class:`ArrayType` or a :class:`MapType`" }, { "name": "to_number", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "format", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Convert string 'col' to a number based on the string format 'format'." }, { "name": "to_timestamp", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "" }, { "name": "to_timestamp", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "format", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "" }, { "name": "to_timestamp", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "format", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Converts a :class:`~pyspark.sql.Column` into :class:`pyspark.sql.types.TimestampType`" }, { "name": "to_timestamp_ltz", "args": [ { "name": "timestamp", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "format", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Parses the `timestamp` with the `format` to a timestamp without time zone." }, { "name": "to_timestamp_ntz", "args": [ { "name": "timestamp", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "format", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Parses the `timestamp` with the `format` to a timestamp without time zone." }, { "name": "to_unix_timestamp", "args": [ { "name": "timestamp", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "format", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the UNIX timestamp of the given time." }, { "name": "to_utc_timestamp", "args": [ { "name": "timestamp", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "tz", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "This is a common function for databases supporting TIMESTAMP WITHOUT TIMEZONE. This function" }, { "name": "to_varchar", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "format", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Convert `col` to a string based on the `format`." }, { "name": "transform", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "f", "default": null, "annotation": "Callable[[Column], Column]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "" }, { "name": "transform", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "f", "default": null, "annotation": "Callable[[Column, Column], Column]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "" }, { "name": "transform", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "f", "default": null, "annotation": "Union[Callable[[Column], Column], Callable[[Column, Column], Column]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns an array of elements after applying a transformation to each element in the input array." }, { "name": "transform_keys", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "f", "default": null, "annotation": "Callable[[Column, Column], Column]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Applies a function to every key-value pair in a map and returns" }, { "name": "transform_values", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "f", "default": null, "annotation": "Callable[[Column, Column], Column]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Applies a function to every key-value pair in a map and returns" }, { "name": "translate", "args": [ { "name": "srcCol", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "matching", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "replace", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "A function translate any character in the `srcCol` by a character in `matching`." }, { "name": "trim", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Trim the spaces from both ends for the specified string column." }, { "name": "trunc", "args": [ { "name": "date", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "format", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns date truncated to the unit specified by the format." }, { "name": "try_add", "args": [ { "name": "left", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "right", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the sum of `left`and `right` and the result is null on overflow." }, { "name": "try_aes_decrypt", "args": [ { "name": "input", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "key", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "mode", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "padding", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "aad", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "This is a special version of `aes_decrypt` that performs the same operation," }, { "name": "try_avg", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the mean calculated from values of a group and the result is null on overflow." }, { "name": "try_divide", "args": [ { "name": "left", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "right", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns `dividend`/`divisor`. It always performs floating point division. Its result is" }, { "name": "try_element_at", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "extraction", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "(array, index) - Returns element of array at given (1-based) index. If Index is 0, Spark will" }, { "name": "try_multiply", "args": [ { "name": "left", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "right", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns `left`*`right` and the result is null on overflow. The acceptable input types are the" }, { "name": "try_subtract", "args": [ { "name": "left", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "right", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns `left`-`right` and the result is null on overflow. The acceptable input types are the" }, { "name": "try_sum", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the sum calculated from values of a group and the result is null on overflow." }, { "name": "try_to_binary", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "format", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "This is a special version of `to_binary` that performs the same operation, but returns a NULL" }, { "name": "try_to_number", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "format", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Convert string 'col' to a number based on the string format `format`. Returns NULL if the" }, { "name": "try_to_timestamp", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "format", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Parses the `col` with the `format` to a timestamp. The function always" }, { "name": "typeof", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Return DDL-formatted type string for the data type of the input." }, { "name": "ucase", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns `str` with all characters changed to uppercase." }, { "name": "udf", "args": [ { "name": "f", "default": null, "annotation": "Callable[..., Any]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "returnType", "default": "StringType()", "annotation": "'DataTypeOrString'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "" }, { "name": "udf", "args": [ { "name": "f", "default": "None", "annotation": "Optional['DataTypeOrString']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "" }, { "name": "udf", "args": [], "doc_preview": "" }, { "name": "udf", "args": [ { "name": "f", "default": "None", "annotation": "Optional[Union[Callable[..., Any], 'DataTypeOrString']]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "returnType", "default": "StringType()", "annotation": "'DataTypeOrString'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Creates a user defined function (UDF)." }, { "name": "udtf", "args": [], "doc_preview": "Creates a user defined table function (UDTF)." }, { "name": "unbase64", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Decodes a BASE64 encoded string column and returns it as a binary column." }, { "name": "unhex", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Inverse of hex. Interprets each pair of characters as a hexadecimal number" }, { "name": "unix_date", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the number of days since 1970-01-01." }, { "name": "unix_micros", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the number of microseconds since 1970-01-01 00:00:00 UTC." }, { "name": "unix_millis", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the number of milliseconds since 1970-01-01 00:00:00 UTC." }, { "name": "unix_seconds", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the number of seconds since 1970-01-01 00:00:00 UTC." }, { "name": "unix_timestamp", "args": [ { "name": "timestamp", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "format", "default": "...", "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "" }, { "name": "unix_timestamp", "args": [], "doc_preview": "" }, { "name": "unix_timestamp", "args": [ { "name": "timestamp", "default": "None", "annotation": "Optional['ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "format", "default": "'yyyy-MM-dd HH:mm:ss'", "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Convert time string with given pattern ('yyyy-MM-dd HH:mm:ss', by default)" }, { "name": "unwrap_udt", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Unwrap UDT data type column into its underlying type." }, { "name": "upper", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Converts a string expression to upper case." }, { "name": "url_decode", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Decodes a `str` in 'application/x-www-form-urlencoded' format" }, { "name": "url_encode", "args": [ { "name": "str", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Translates a string into 'application/x-www-form-urlencoded' format" }, { "name": "user", "args": [], "doc_preview": "Returns the current database." }, { "name": "var_pop", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the population variance of the values in a group." }, { "name": "var_samp", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: returns the unbiased sample variance of" }, { "name": "variance", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Aggregate function: alias for var_samp" }, { "name": "version", "args": [], "doc_preview": "Returns the Spark version. The string contains 2 fields, the first being a release version" }, { "name": "weekday", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the day of the week for date/timestamp (0 = Monday, 1 = Tuesday, ..., 6 = Sunday)." }, { "name": "weekofyear", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Extract the week number of a given date as integer." }, { "name": "when", "args": [ { "name": "condition", "default": null, "annotation": "Column", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "value", "default": null, "annotation": "Any", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Evaluates a list of conditions and returns one of multiple possible result expressions." }, { "name": "width_bucket", "args": [ { "name": "v", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "min", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "max", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "numBucket", "default": null, "annotation": "Union['ColumnOrName', int]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the bucket number into which the value of this expression would fall" }, { "name": "window", "args": [ { "name": "timeColumn", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "windowDuration", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "slideDuration", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "startTime", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Bucketize rows into one or more time windows given a timestamp specifying column. Window" }, { "name": "window_time", "args": [ { "name": "windowColumn", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes the event time from a window column. The column window values are produced" }, { "name": "xpath", "args": [ { "name": "xml", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "path", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a string array of values within the nodes of xml that match the XPath expression." }, { "name": "xpath_boolean", "args": [ { "name": "xml", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "path", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns true if the XPath expression evaluates to true, or if a matching node is found." }, { "name": "xpath_double", "args": [ { "name": "xml", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "path", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a double value, the value zero if no match is found," }, { "name": "xpath_float", "args": [ { "name": "xml", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "path", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a float value, the value zero if no match is found," }, { "name": "xpath_int", "args": [ { "name": "xml", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "path", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns an integer value, or the value zero if no match is found," }, { "name": "xpath_long", "args": [ { "name": "xml", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "path", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a long integer value, or the value zero if no match is found," }, { "name": "xpath_number", "args": [ { "name": "xml", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "path", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a double value, the value zero if no match is found," }, { "name": "xpath_short", "args": [ { "name": "xml", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "path", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a short integer value, or the value zero if no match is found," }, { "name": "xpath_string", "args": [ { "name": "xml", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "path", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the text contents of the first xml node that matches the XPath expression." }, { "name": "xxhash64", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Calculates the hash code of given columns using the 64-bit variant of the xxHash algorithm," }, { "name": "year", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Extract the year of a given date/timestamp as integer." }, { "name": "years", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Partition transform function: A transform for timestamps and dates" }, { "name": "zip_with", "args": [ { "name": "left", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "right", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "f", "default": null, "annotation": "Callable[[Column, Column], Column]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Merge two given arrays, element-wise, into a single array using a function." } ], "DataFrame": [ { "name": "agg", "args": [ { "name": "exprs", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Aggregate on the entire :class:`DataFrame` without groups", "class": "DataFrame" }, { "name": "alias", "args": [ { "name": "alias", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a new :class:`DataFrame` with an alias set.", "class": "DataFrame" }, { "name": "approxQuantile", "args": [ { "name": "col", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "probabilities", "default": null, "annotation": "Union[List[float], Tuple[float]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "relativeError", "default": null, "annotation": "float", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "approxQuantile", "args": [ { "name": "col", "default": null, "annotation": "Union[List[str], Tuple[str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "probabilities", "default": null, "annotation": "Union[List[float], Tuple[float]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "relativeError", "default": null, "annotation": "float", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "approxQuantile", "args": [ { "name": "col", "default": null, "annotation": "Union[str, List[str], Tuple[str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "probabilities", "default": null, "annotation": "Union[List[float], Tuple[float]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "relativeError", "default": null, "annotation": "float", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Calculates the approximate quantiles of numerical columns of a", "class": "DataFrame" }, { "name": "cache", "args": [], "doc_preview": "Persists the :class:`DataFrame` with the default storage level (`MEMORY_AND_DISK`).", "class": "DataFrame" }, { "name": "checkpoint", "args": [ { "name": "eager", "default": "True", "annotation": "bool", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a checkpointed version of this :class:`DataFrame`. Checkpointing can be used to", "class": "DataFrame" }, { "name": "coalesce", "args": [ { "name": "numPartitions", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a new :class:`DataFrame` that has exactly `numPartitions` partitions.", "class": "DataFrame" }, { "name": "colRegex", "args": [ { "name": "colName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Selects column based on the column name specified as a regex and returns it", "class": "DataFrame" }, { "name": "collect", "args": [], "doc_preview": "Returns all the records as a list of :class:`Row`.", "class": "DataFrame" }, { "name": "columns", "args": [], "doc_preview": "Retrieves the names of all columns in the :class:`DataFrame` as a list.", "class": "DataFrame" }, { "name": "corr", "args": [ { "name": "col1", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col2", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "method", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Calculates the correlation of two columns of a :class:`DataFrame` as a double value.", "class": "DataFrame" }, { "name": "count", "args": [], "doc_preview": "Returns the number of rows in this :class:`DataFrame`.", "class": "DataFrame" }, { "name": "cov", "args": [ { "name": "col1", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col2", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Calculate the sample covariance for the given columns, specified by their names, as a", "class": "DataFrame" }, { "name": "createGlobalTempView", "args": [ { "name": "name", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Creates a global temporary view with this :class:`DataFrame`.", "class": "DataFrame" }, { "name": "createOrReplaceGlobalTempView", "args": [ { "name": "name", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Creates or replaces a global temporary view using the given name.", "class": "DataFrame" }, { "name": "createOrReplaceTempView", "args": [ { "name": "name", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Creates or replaces a local temporary view with this :class:`DataFrame`.", "class": "DataFrame" }, { "name": "createTempView", "args": [ { "name": "name", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Creates a local temporary view with this :class:`DataFrame`.", "class": "DataFrame" }, { "name": "crossJoin", "args": [ { "name": "other", "default": null, "annotation": "'DataFrame'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the cartesian product with another :class:`DataFrame`.", "class": "DataFrame" }, { "name": "crosstab", "args": [ { "name": "col1", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col2", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Computes a pair-wise frequency table of the given columns. Also known as a contingency", "class": "DataFrame" }, { "name": "cube", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "cube", "args": [ { "name": "__cols", "default": null, "annotation": "Union[List[Column], List[str]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "cube", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Create a multi-dimensional cube for the current :class:`DataFrame` using", "class": "DataFrame" }, { "name": "describe", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Computes basic statistics for numeric and string columns.", "class": "DataFrame" }, { "name": "distinct", "args": [], "doc_preview": "Returns a new :class:`DataFrame` containing the distinct rows in this :class:`DataFrame`.", "class": "DataFrame" }, { "name": "drop", "args": [ { "name": "cols", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "drop", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "drop", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Returns a new :class:`DataFrame` without specified columns.", "class": "DataFrame" }, { "name": "dropDuplicates", "args": [ { "name": "subset", "default": "None", "annotation": "Optional[List[str]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Return a new :class:`DataFrame` with duplicate rows removed,", "class": "DataFrame" }, { "name": "dropDuplicatesWithinWatermark", "args": [ { "name": "subset", "default": "None", "annotation": "Optional[List[str]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Return a new :class:`DataFrame` with duplicate rows removed,", "class": "DataFrame" }, { "name": "dropna", "args": [ { "name": "how", "default": "'any'", "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "thresh", "default": "None", "annotation": "Optional[int]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "subset", "default": "None", "annotation": "Optional[Union[str, Tuple[str, ...], List[str]]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a new :class:`DataFrame` omitting rows with null values.", "class": "DataFrame" }, { "name": "dtypes", "args": [], "doc_preview": "Returns all column names and their data types as a list.", "class": "DataFrame" }, { "name": "exceptAll", "args": [ { "name": "other", "default": null, "annotation": "'DataFrame'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Return a new :class:`DataFrame` containing rows in this :class:`DataFrame` but", "class": "DataFrame" }, { "name": "explain", "args": [ { "name": "extended", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "mode", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Prints the (logical and physical) plans to the console for debugging purposes.", "class": "DataFrame" }, { "name": "fillna", "args": [ { "name": "value", "default": null, "annotation": "'LiteralType'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "subset", "default": "...", "annotation": "Optional[Union[str, Tuple[str, ...], List[str]]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "fillna", "args": [ { "name": "value", "default": null, "annotation": "Dict[str, 'LiteralType']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "fillna", "args": [ { "name": "value", "default": null, "annotation": "Union['LiteralType', Dict[str, 'LiteralType']]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "subset", "default": "None", "annotation": "Optional[Union[str, Tuple[str, ...], List[str]]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Replace null values, alias for ``na.fill()``.", "class": "DataFrame" }, { "name": "filter", "args": [ { "name": "condition", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Filters rows using the given condition.", "class": "DataFrame" }, { "name": "first", "args": [], "doc_preview": "Returns the first row as a :class:`Row`.", "class": "DataFrame" }, { "name": "foreach", "args": [ { "name": "f", "default": null, "annotation": "Callable[[Row], None]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Applies the ``f`` function to all :class:`Row` of this :class:`DataFrame`.", "class": "DataFrame" }, { "name": "foreachPartition", "args": [ { "name": "f", "default": null, "annotation": "Callable[[Iterator[Row]], None]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Applies the ``f`` function to each partition of this :class:`DataFrame`.", "class": "DataFrame" }, { "name": "freqItems", "args": [ { "name": "cols", "default": null, "annotation": "Union[List[str], Tuple[str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "support", "default": "None", "annotation": "Optional[float]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Finding frequent items for columns, possibly with false positives. Using the", "class": "DataFrame" }, { "name": "groupBy", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "groupBy", "args": [ { "name": "__cols", "default": null, "annotation": "Union[List[Column], List[str]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "groupBy", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Groups the :class:`DataFrame` using the specified columns,", "class": "DataFrame" }, { "name": "head", "args": [], "doc_preview": "", "class": "DataFrame" }, { "name": "head", "args": [ { "name": "n", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "head", "args": [ { "name": "n", "default": "None", "annotation": "Optional[int]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the first ``n`` rows.", "class": "DataFrame" }, { "name": "hint", "args": [ { "name": "name", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "parameters", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Specifies some hint on the current :class:`DataFrame`.", "class": "DataFrame" }, { "name": "inputFiles", "args": [], "doc_preview": "Returns a best-effort snapshot of the files that compose this :class:`DataFrame`.", "class": "DataFrame" }, { "name": "intersect", "args": [ { "name": "other", "default": null, "annotation": "'DataFrame'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Return a new :class:`DataFrame` containing rows only in", "class": "DataFrame" }, { "name": "intersectAll", "args": [ { "name": "other", "default": null, "annotation": "'DataFrame'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Return a new :class:`DataFrame` containing rows in both this :class:`DataFrame`", "class": "DataFrame" }, { "name": "isEmpty", "args": [], "doc_preview": "Checks if the :class:`DataFrame` is empty and returns a boolean value.", "class": "DataFrame" }, { "name": "isLocal", "args": [], "doc_preview": "Returns ``True`` if the :func:`collect` and :func:`take` methods can be run locally", "class": "DataFrame" }, { "name": "isStreaming", "args": [], "doc_preview": "Returns ``True`` if this :class:`DataFrame` contains one or more sources that", "class": "DataFrame" }, { "name": "join", "args": [ { "name": "other", "default": null, "annotation": "'DataFrame'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "on", "default": "None", "annotation": "Optional[Union[str, List[str], Column, List[Column]]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "how", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Joins with another :class:`DataFrame`, using the given join expression.", "class": "DataFrame" }, { "name": "limit", "args": [ { "name": "num", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Limits the result count to the number specified.", "class": "DataFrame" }, { "name": "localCheckpoint", "args": [ { "name": "eager", "default": "True", "annotation": "bool", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a locally checkpointed version of this :class:`DataFrame`. Checkpointing can be", "class": "DataFrame" }, { "name": "melt", "args": [ { "name": "ids", "default": null, "annotation": "Union['ColumnOrName', List['ColumnOrName'], Tuple['ColumnOrName', ...]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "values", "default": null, "annotation": "Optional[Union['ColumnOrName', List['ColumnOrName'], Tuple['ColumnOrName', ...]]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "variableColumnName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "valueColumnName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Unpivot a DataFrame from wide format to long format, optionally leaving", "class": "DataFrame" }, { "name": "na", "args": [], "doc_preview": "Returns a :class:`DataFrameNaFunctions` for handling missing values.", "class": "DataFrame" }, { "name": "observe", "args": [ { "name": "observation", "default": null, "annotation": "Union['Observation', str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "exprs", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Define (named) metrics to observe on the DataFrame. This method returns an 'observed'", "class": "DataFrame" }, { "name": "offset", "args": [ { "name": "num", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a new :class: `DataFrame` by skipping the first `n` rows.", "class": "DataFrame" }, { "name": "pandas_api", "args": [ { "name": "index_col", "default": "None", "annotation": "Optional[Union[str, List[str]]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Converts the existing DataFrame into a pandas-on-Spark DataFrame.", "class": "DataFrame" }, { "name": "persist", "args": [ { "name": "storageLevel", "default": "StorageLevel.MEMORY_AND_DISK_DESER", "annotation": "StorageLevel", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Sets the storage level to persist the contents of the :class:`DataFrame` across", "class": "DataFrame" }, { "name": "printSchema", "args": [ { "name": "level", "default": "None", "annotation": "Optional[int]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Prints out the schema in the tree format.", "class": "DataFrame" }, { "name": "randomSplit", "args": [ { "name": "weights", "default": null, "annotation": "List[float]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "seed", "default": "None", "annotation": "Optional[int]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Randomly splits this :class:`DataFrame` with the provided weights.", "class": "DataFrame" }, { "name": "rdd", "args": [], "doc_preview": "Returns the content as an :class:`pyspark.RDD` of :class:`Row`.", "class": "DataFrame" }, { "name": "registerTempTable", "args": [ { "name": "name", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Registers this :class:`DataFrame` as a temporary table using the given name.", "class": "DataFrame" }, { "name": "repartition", "args": [ { "name": "numPartitions", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "repartition", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "repartition", "args": [ { "name": "numPartitions", "default": null, "annotation": "Union[int, 'ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Returns a new :class:`DataFrame` partitioned by the given partitioning expressions. The", "class": "DataFrame" }, { "name": "repartitionByRange", "args": [ { "name": "numPartitions", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "repartitionByRange", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "repartitionByRange", "args": [ { "name": "numPartitions", "default": null, "annotation": "Union[int, 'ColumnOrName']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Returns a new :class:`DataFrame` partitioned by the given partitioning expressions. The", "class": "DataFrame" }, { "name": "replace", "args": [ { "name": "to_replace", "default": null, "annotation": "'LiteralType'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "value", "default": null, "annotation": "'OptionalPrimitiveType'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "subset", "default": "...", "annotation": "Optional[List[str]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "replace", "args": [ { "name": "to_replace", "default": null, "annotation": "List['LiteralType']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "value", "default": null, "annotation": "List['OptionalPrimitiveType']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "subset", "default": "...", "annotation": "Optional[List[str]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "replace", "args": [ { "name": "to_replace", "default": null, "annotation": "Dict['LiteralType', 'OptionalPrimitiveType']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "subset", "default": "...", "annotation": "Optional[List[str]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "replace", "args": [ { "name": "to_replace", "default": null, "annotation": "List['LiteralType']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "value", "default": null, "annotation": "'OptionalPrimitiveType'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "subset", "default": "...", "annotation": "Optional[List[str]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "replace", "args": [ { "name": "to_replace", "default": null, "annotation": "Union['LiteralType', List['LiteralType'], Dict['LiteralType', 'OptionalPrimitiveType']]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "value", "default": "_NoValue", "annotation": "Optional[Union['OptionalPrimitiveType', List['OptionalPrimitiveType'], _NoValueType]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "subset", "default": "None", "annotation": "Optional[List[str]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a new :class:`DataFrame` replacing a value with another value.", "class": "DataFrame" }, { "name": "rollup", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "rollup", "args": [ { "name": "__cols", "default": null, "annotation": "Union[List[Column], List[str]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "rollup", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Create a multi-dimensional rollup for the current :class:`DataFrame` using", "class": "DataFrame" }, { "name": "sameSemantics", "args": [ { "name": "other", "default": null, "annotation": "'DataFrame'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns `True` when the logical query plans inside both :class:`DataFrame`\\s are equal and", "class": "DataFrame" }, { "name": "sample", "args": [ { "name": "fraction", "default": null, "annotation": "float", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "seed", "default": "...", "annotation": "Optional[int]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "sample", "args": [ { "name": "withReplacement", "default": null, "annotation": "Optional[bool]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "fraction", "default": null, "annotation": "float", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "seed", "default": "...", "annotation": "Optional[int]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "sample", "args": [ { "name": "withReplacement", "default": "None", "annotation": "Optional[Union[float, bool]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "fraction", "default": "None", "annotation": "Optional[Union[int, float]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "seed", "default": "None", "annotation": "Optional[int]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a sampled subset of this :class:`DataFrame`.", "class": "DataFrame" }, { "name": "sampleBy", "args": [ { "name": "col", "default": null, "annotation": "'ColumnOrName'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "fractions", "default": null, "annotation": "Dict[Any, float]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "seed", "default": "None", "annotation": "Optional[int]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a stratified sample without replacement based on the", "class": "DataFrame" }, { "name": "schema", "args": [], "doc_preview": "Returns the schema of this :class:`DataFrame` as a :class:`pyspark.sql.types.StructType`.", "class": "DataFrame" }, { "name": "select", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "select", "args": [ { "name": "__cols", "default": null, "annotation": "Union[List[Column], List[str]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "select", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Projects a set of expressions and returns a new :class:`DataFrame`.", "class": "DataFrame" }, { "name": "selectExpr", "args": [ { "name": "expr", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "selectExpr", "args": [ { "name": "expr", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "selectExpr", "args": [ { "name": "expr", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Projects a set of SQL expressions and returns a new :class:`DataFrame`.", "class": "DataFrame" }, { "name": "semanticHash", "args": [], "doc_preview": "Returns a hash code of the logical query plan against this :class:`DataFrame`.", "class": "DataFrame" }, { "name": "show", "args": [ { "name": "n", "default": "20", "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "truncate", "default": "True", "annotation": "Union[bool, int]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "vertical", "default": "False", "annotation": "bool", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Prints the first ``n`` rows to the console.", "class": "DataFrame" }, { "name": "sort", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" }, { "name": "kwargs", "default": null, "annotation": null, "kind": "VAR_KEYWORD" } ], "doc_preview": "Returns a new :class:`DataFrame` sorted by the specified column(s).", "class": "DataFrame" }, { "name": "sortWithinPartitions", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" }, { "name": "kwargs", "default": null, "annotation": null, "kind": "VAR_KEYWORD" } ], "doc_preview": "Returns a new :class:`DataFrame` with each partition sorted by the specified column(s).", "class": "DataFrame" }, { "name": "sparkSession", "args": [], "doc_preview": "Returns Spark session that created this :class:`DataFrame`.", "class": "DataFrame" }, { "name": "sql_ctx", "args": [], "doc_preview": "", "class": "DataFrame" }, { "name": "stat", "args": [], "doc_preview": "Returns a :class:`DataFrameStatFunctions` for statistic functions.", "class": "DataFrame" }, { "name": "storageLevel", "args": [], "doc_preview": "Get the :class:`DataFrame`'s current storage level.", "class": "DataFrame" }, { "name": "subtract", "args": [ { "name": "other", "default": null, "annotation": "'DataFrame'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Return a new :class:`DataFrame` containing rows in this :class:`DataFrame`", "class": "DataFrame" }, { "name": "summary", "args": [ { "name": "statistics", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Computes specified statistics for numeric and string columns. Available statistics are:", "class": "DataFrame" }, { "name": "tail", "args": [ { "name": "num", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the last ``num`` rows as a :class:`list` of :class:`Row`.", "class": "DataFrame" }, { "name": "take", "args": [ { "name": "num", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the first ``num`` rows as a :class:`list` of :class:`Row`.", "class": "DataFrame" }, { "name": "to", "args": [ { "name": "schema", "default": null, "annotation": "StructType", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a new :class:`DataFrame` where each row is reconciled to match the specified", "class": "DataFrame" }, { "name": "toDF", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Returns a new :class:`DataFrame` that with new specified column names", "class": "DataFrame" }, { "name": "toJSON", "args": [ { "name": "use_unicode", "default": "True", "annotation": "bool", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Converts a :class:`DataFrame` into a :class:`RDD` of string.", "class": "DataFrame" }, { "name": "toLocalIterator", "args": [ { "name": "prefetchPartitions", "default": "False", "annotation": "bool", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns an iterator that contains all of the rows in this :class:`DataFrame`.", "class": "DataFrame" }, { "name": "to_koalas", "args": [ { "name": "index_col", "default": "None", "annotation": "Optional[Union[str, List[str]]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "to_pandas_on_spark", "args": [ { "name": "index_col", "default": "None", "annotation": "Optional[Union[str, List[str]]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "DataFrame" }, { "name": "transform", "args": [ { "name": "func", "default": null, "annotation": "Callable[..., 'DataFrame']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "args", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" }, { "name": "kwargs", "default": null, "annotation": null, "kind": "VAR_KEYWORD" } ], "doc_preview": "Returns a new :class:`DataFrame`. Concise syntax for chaining custom transformations.", "class": "DataFrame" }, { "name": "union", "args": [ { "name": "other", "default": null, "annotation": "'DataFrame'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Return a new :class:`DataFrame` containing the union of rows in this and another", "class": "DataFrame" }, { "name": "unionAll", "args": [ { "name": "other", "default": null, "annotation": "'DataFrame'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Return a new :class:`DataFrame` containing the union of rows in this and another", "class": "DataFrame" }, { "name": "unionByName", "args": [ { "name": "other", "default": null, "annotation": "'DataFrame'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "allowMissingColumns", "default": "False", "annotation": "bool", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a new :class:`DataFrame` containing union of rows in this and another", "class": "DataFrame" }, { "name": "unpersist", "args": [ { "name": "blocking", "default": "False", "annotation": "bool", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Marks the :class:`DataFrame` as non-persistent, and remove all blocks for it from", "class": "DataFrame" }, { "name": "unpivot", "args": [ { "name": "ids", "default": null, "annotation": "Union['ColumnOrName', List['ColumnOrName'], Tuple['ColumnOrName', ...]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "values", "default": null, "annotation": "Optional[Union['ColumnOrName', List['ColumnOrName'], Tuple['ColumnOrName', ...]]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "variableColumnName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "valueColumnName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Unpivot a DataFrame from wide format to long format, optionally leaving", "class": "DataFrame" }, { "name": "withColumn", "args": [ { "name": "colName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col", "default": null, "annotation": "Column", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a new :class:`DataFrame` by adding a column or replacing the", "class": "DataFrame" }, { "name": "withColumnRenamed", "args": [ { "name": "existing", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "new", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a new :class:`DataFrame` by renaming an existing column.", "class": "DataFrame" }, { "name": "withColumns", "args": [ { "name": "colsMap", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Returns a new :class:`DataFrame` by adding multiple columns or replacing the", "class": "DataFrame" }, { "name": "withColumnsRenamed", "args": [ { "name": "colsMap", "default": null, "annotation": "Dict[str, str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a new :class:`DataFrame` by renaming multiple columns.", "class": "DataFrame" }, { "name": "withMetadata", "args": [ { "name": "columnName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "metadata", "default": null, "annotation": "Dict[str, Any]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a new :class:`DataFrame` by updating an existing column with metadata.", "class": "DataFrame" }, { "name": "withWatermark", "args": [ { "name": "eventTime", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "delayThreshold", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Defines an event time watermark for this :class:`DataFrame`. A watermark tracks a point", "class": "DataFrame" }, { "name": "write", "args": [], "doc_preview": "Interface for saving the content of the non-streaming :class:`DataFrame` out into external", "class": "DataFrame" }, { "name": "writeStream", "args": [], "doc_preview": "Interface for saving the content of the streaming :class:`DataFrame` out into external", "class": "DataFrame" }, { "name": "writeTo", "args": [ { "name": "table", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Create a write configuration builder for v2 sources.", "class": "DataFrame" } ], "Column": [ { "name": "alias", "args": [ { "name": "alias", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" }, { "name": "kwargs", "default": null, "annotation": null, "kind": "VAR_KEYWORD" } ], "doc_preview": "Returns this column aliased with a new name or names (in the case of expressions that", "class": "Column" }, { "name": "between", "args": [ { "name": "lowerBound", "default": null, "annotation": "Union['Column', 'LiteralType', 'DateTimeLiteral', 'DecimalLiteral']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "upperBound", "default": null, "annotation": "Union['Column', 'LiteralType', 'DateTimeLiteral', 'DecimalLiteral']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "True if the current column is between the lower bound and upper bound, inclusive.", "class": "Column" }, { "name": "cast", "args": [ { "name": "dataType", "default": null, "annotation": "Union[DataType, str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Casts the column into type ``dataType``.", "class": "Column" }, { "name": "dropFields", "args": [ { "name": "fieldNames", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "An expression that drops fields in :class:`StructType` by name.", "class": "Column" }, { "name": "getField", "args": [ { "name": "name", "default": null, "annotation": "Any", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "An expression that gets a field by name in a :class:`StructType`.", "class": "Column" }, { "name": "getItem", "args": [ { "name": "key", "default": null, "annotation": "Any", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "An expression that gets an item at position ``ordinal`` out of a list,", "class": "Column" }, { "name": "ilike", "args": [ { "name": "other", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "SQL ILIKE expression (case insensitive LIKE). Returns a boolean :class:`Column`", "class": "Column" }, { "name": "isin", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "A boolean expression that is evaluated to true if the value of this", "class": "Column" }, { "name": "like", "args": [ { "name": "other", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "SQL like expression. Returns a boolean :class:`Column` based on a SQL LIKE match.", "class": "Column" }, { "name": "otherwise", "args": [ { "name": "value", "default": null, "annotation": "Any", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Evaluates a list of conditions and returns one of multiple possible result expressions.", "class": "Column" }, { "name": "over", "args": [ { "name": "window", "default": null, "annotation": "'WindowSpec'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Define a windowing column.", "class": "Column" }, { "name": "rlike", "args": [ { "name": "other", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "SQL RLIKE expression (LIKE with Regex). Returns a boolean :class:`Column` based on a regex", "class": "Column" }, { "name": "substr", "args": [ { "name": "startPos", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "length", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "Column" }, { "name": "substr", "args": [ { "name": "startPos", "default": null, "annotation": "'Column'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "length", "default": null, "annotation": "'Column'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "Column" }, { "name": "substr", "args": [ { "name": "startPos", "default": null, "annotation": "Union[int, 'Column']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "length", "default": null, "annotation": "Union[int, 'Column']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Return a :class:`Column` which is a substring of the column.", "class": "Column" }, { "name": "when", "args": [ { "name": "condition", "default": null, "annotation": "'Column'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "value", "default": null, "annotation": "Any", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Evaluates a list of conditions and returns one of multiple possible result expressions.", "class": "Column" }, { "name": "withField", "args": [ { "name": "fieldName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col", "default": null, "annotation": "'Column'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "An expression that adds/replaces a field in :class:`StructType` by name.", "class": "Column" } ], "GroupedData": [ { "name": "agg", "args": [ { "name": "exprs", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "", "class": "GroupedData" }, { "name": "agg", "args": [ { "name": "__exprs", "default": null, "annotation": "Dict[str, str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "GroupedData" }, { "name": "agg", "args": [ { "name": "exprs", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Compute aggregates and returns the result as a :class:`DataFrame`.", "class": "GroupedData" }, { "name": "avg", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Computes average values for each numeric columns for each group.", "class": "GroupedData" }, { "name": "count", "args": [], "doc_preview": "Counts the number of records for each group.", "class": "GroupedData" }, { "name": "max", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Computes the max value for each numeric columns for each group.", "class": "GroupedData" }, { "name": "mean", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Computes average values for each numeric columns for each group.", "class": "GroupedData" }, { "name": "min", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Computes the min value for each numeric column for each group.", "class": "GroupedData" }, { "name": "pivot", "args": [ { "name": "pivot_col", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "values", "default": "None", "annotation": "Optional[List['LiteralType']]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Pivots a column of the current :class:`DataFrame` and perform the specified aggregation.", "class": "GroupedData" }, { "name": "sum", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Computes the sum for each numeric columns for each group.", "class": "GroupedData" } ], "SparkSession": [ { "name": "active", "args": [], "doc_preview": "Returns the active or default :class:`SparkSession` for the current thread, returned by", "class": "SparkSession" }, { "name": "addArtifacts", "args": [ { "name": "path", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Add artifact(s) to the client session. Currently only local files are supported.", "class": "SparkSession" }, { "name": "addTag", "args": [ { "name": "tag", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Add a tag to be assigned to all the operations started by this thread in this session.", "class": "SparkSession" }, { "name": "builder", "args": [], "doc_preview": "Creates a :class:`Builder` for constructing a :class:`SparkSession`.", "class": "SparkSession" }, { "name": "catalog", "args": [], "doc_preview": "Interface through which the user may create, drop, alter or query underlying", "class": "SparkSession" }, { "name": "clearTags", "args": [], "doc_preview": "Clear the current thread's operation tags.", "class": "SparkSession" }, { "name": "client", "args": [], "doc_preview": "Gives access to the Spark Connect client. In normal cases this is not necessary to be used", "class": "SparkSession" }, { "name": "conf", "args": [], "doc_preview": "Runtime configuration interface for Spark.", "class": "SparkSession" }, { "name": "copyFromLocalToFs", "args": [ { "name": "local_path", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "dest_path", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Copy file from local to cloud storage file system.", "class": "SparkSession" }, { "name": "createDataFrame", "args": [ { "name": "data", "default": null, "annotation": "Iterable['RowLike']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "schema", "default": "...", "annotation": "Union[List[str], Tuple[str, ...]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "samplingRatio", "default": "...", "annotation": "Optional[float]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "SparkSession" }, { "name": "createDataFrame", "args": [ { "name": "data", "default": null, "annotation": "'RDD[RowLike]'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "schema", "default": "...", "annotation": "Union[List[str], Tuple[str, ...]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "samplingRatio", "default": "...", "annotation": "Optional[float]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "SparkSession" }, { "name": "createDataFrame", "args": [ { "name": "data", "default": null, "annotation": "Iterable['RowLike']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "schema", "default": null, "annotation": "Union[StructType, str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "SparkSession" }, { "name": "createDataFrame", "args": [ { "name": "data", "default": null, "annotation": "'RDD[RowLike]'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "schema", "default": null, "annotation": "Union[StructType, str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "SparkSession" }, { "name": "createDataFrame", "args": [ { "name": "data", "default": null, "annotation": "'RDD[AtomicValue]'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "schema", "default": null, "annotation": "Union[AtomicType, str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "verifySchema", "default": "...", "annotation": "bool", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "SparkSession" }, { "name": "createDataFrame", "args": [ { "name": "data", "default": null, "annotation": "Iterable['AtomicValue']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "schema", "default": null, "annotation": "Union[AtomicType, str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "verifySchema", "default": "...", "annotation": "bool", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "SparkSession" }, { "name": "createDataFrame", "args": [ { "name": "data", "default": null, "annotation": "'PandasDataFrameLike'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "samplingRatio", "default": "...", "annotation": "Optional[float]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "SparkSession" }, { "name": "createDataFrame", "args": [ { "name": "data", "default": null, "annotation": "'PandasDataFrameLike'", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "schema", "default": null, "annotation": "Union[StructType, str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "verifySchema", "default": "...", "annotation": "bool", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "SparkSession" }, { "name": "createDataFrame", "args": [ { "name": "data", "default": null, "annotation": "Union[RDD[Any], Iterable[Any], 'PandasDataFrameLike', 'ArrayLike']", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "schema", "default": "None", "annotation": "Optional[Union[AtomicType, StructType, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "samplingRatio", "default": "None", "annotation": "Optional[float]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "verifySchema", "default": "True", "annotation": "bool", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Creates a :class:`DataFrame` from an :class:`RDD`, a list, a :class:`pandas.DataFrame`", "class": "SparkSession" }, { "name": "getActiveSession", "args": [], "doc_preview": "Returns the active :class:`SparkSession` for the current thread, returned by the builder", "class": "SparkSession" }, { "name": "getTags", "args": [], "doc_preview": "Get the tags that are currently set to be assigned to all the operations started by this", "class": "SparkSession" }, { "name": "interruptAll", "args": [], "doc_preview": "Interrupt all operations of this session currently running on the connected server.", "class": "SparkSession" }, { "name": "interruptOperation", "args": [ { "name": "op_id", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Interrupt an operation of this session with the given operationId.", "class": "SparkSession" }, { "name": "interruptTag", "args": [ { "name": "tag", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Interrupt all operations of this session with the given operation tag.", "class": "SparkSession" }, { "name": "newSession", "args": [], "doc_preview": "Returns a new :class:`SparkSession` as new session, that has separate SQLConf,", "class": "SparkSession" }, { "name": "range", "args": [ { "name": "start", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "end", "default": "None", "annotation": "Optional[int]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "step", "default": "1", "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "numPartitions", "default": "None", "annotation": "Optional[int]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Create a :class:`DataFrame` with single :class:`pyspark.sql.types.LongType` column named", "class": "SparkSession" }, { "name": "read", "args": [], "doc_preview": "Returns a :class:`DataFrameReader` that can be used to read data", "class": "SparkSession" }, { "name": "readStream", "args": [], "doc_preview": "Returns a :class:`DataStreamReader` that can be used to read data streams", "class": "SparkSession" }, { "name": "removeTag", "args": [ { "name": "tag", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Remove a tag previously added to be assigned to all the operations started by this thread in", "class": "SparkSession" }, { "name": "sparkContext", "args": [], "doc_preview": "Returns the underlying :class:`SparkContext`.", "class": "SparkSession" }, { "name": "sql", "args": [ { "name": "sqlQuery", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "args", "default": "None", "annotation": "Optional[Union[Dict[str, Any], List]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "kwargs", "default": null, "annotation": null, "kind": "VAR_KEYWORD" } ], "doc_preview": "Returns a :class:`DataFrame` representing the result of the given query.", "class": "SparkSession" }, { "name": "stop", "args": [], "doc_preview": "Stop the underlying :class:`SparkContext`.", "class": "SparkSession" }, { "name": "streams", "args": [], "doc_preview": "Returns a :class:`StreamingQueryManager` that allows managing all the", "class": "SparkSession" }, { "name": "table", "args": [ { "name": "tableName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the specified table as a :class:`DataFrame`.", "class": "SparkSession" }, { "name": "udf", "args": [], "doc_preview": "Returns a :class:`UDFRegistration` for UDF registration.", "class": "SparkSession" }, { "name": "udtf", "args": [], "doc_preview": "Returns a :class:`UDTFRegistration` for UDTF registration.", "class": "SparkSession" }, { "name": "version", "args": [], "doc_preview": "The version of Spark on which this application is running.", "class": "SparkSession" } ], "DataFrameReader": [ { "name": "csv", "args": [ { "name": "path", "default": null, "annotation": "PathOrPaths", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "schema", "default": "None", "annotation": "Optional[Union[StructType, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "sep", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "encoding", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "quote", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "escape", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "comment", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "header", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "inferSchema", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "ignoreLeadingWhiteSpace", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "ignoreTrailingWhiteSpace", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "nullValue", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "nanValue", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "positiveInf", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "negativeInf", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "dateFormat", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "timestampFormat", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "maxColumns", "default": "None", "annotation": "Optional[Union[int, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "maxCharsPerColumn", "default": "None", "annotation": "Optional[Union[int, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "maxMalformedLogPerPartition", "default": "None", "annotation": "Optional[Union[int, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "mode", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "columnNameOfCorruptRecord", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "multiLine", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "charToEscapeQuoteEscaping", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "samplingRatio", "default": "None", "annotation": "Optional[Union[float, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "enforceSchema", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "emptyValue", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "locale", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "lineSep", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "pathGlobFilter", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "recursiveFileLookup", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "modifiedBefore", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "modifiedAfter", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "unescapedQuoteHandling", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Loads a CSV file and returns the result as a :class:`DataFrame`.", "class": "DataFrameReader" }, { "name": "format", "args": [ { "name": "source", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Specifies the input data source format.", "class": "DataFrameReader" }, { "name": "jdbc", "args": [ { "name": "url", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "table", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "DataFrameReader" }, { "name": "jdbc", "args": [ { "name": "url", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "table", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "column", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "lowerBound", "default": null, "annotation": "Union[int, str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "upperBound", "default": null, "annotation": "Union[int, str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "numPartitions", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "DataFrameReader" }, { "name": "jdbc", "args": [ { "name": "url", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "table", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "DataFrameReader" }, { "name": "jdbc", "args": [ { "name": "url", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "table", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "column", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "lowerBound", "default": "None", "annotation": "Optional[Union[int, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "upperBound", "default": "None", "annotation": "Optional[Union[int, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "numPartitions", "default": "None", "annotation": "Optional[int]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "predicates", "default": "None", "annotation": "Optional[List[str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "properties", "default": "None", "annotation": "Optional[Dict[str, str]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Construct a :class:`DataFrame` representing the database table named ``table``", "class": "DataFrameReader" }, { "name": "json", "args": [ { "name": "path", "default": null, "annotation": "Union[str, List[str], RDD[str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "schema", "default": "None", "annotation": "Optional[Union[StructType, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "primitivesAsString", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "prefersDecimal", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "allowComments", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "allowUnquotedFieldNames", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "allowSingleQuotes", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "allowNumericLeadingZero", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "allowBackslashEscapingAnyCharacter", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "mode", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "columnNameOfCorruptRecord", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "dateFormat", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "timestampFormat", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "multiLine", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "allowUnquotedControlChars", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "lineSep", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "samplingRatio", "default": "None", "annotation": "Optional[Union[float, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "dropFieldIfAllNull", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "encoding", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "locale", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "pathGlobFilter", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "recursiveFileLookup", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "modifiedBefore", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "modifiedAfter", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "allowNonNumericNumbers", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Loads JSON files and returns the results as a :class:`DataFrame`.", "class": "DataFrameReader" }, { "name": "load", "args": [ { "name": "path", "default": "None", "annotation": "Optional[PathOrPaths]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "format", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "schema", "default": "None", "annotation": "Optional[Union[StructType, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "options", "default": null, "annotation": null, "kind": "VAR_KEYWORD" } ], "doc_preview": "Loads data from a data source and returns it as a :class:`DataFrame`.", "class": "DataFrameReader" }, { "name": "option", "args": [ { "name": "key", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "value", "default": null, "annotation": "'OptionalPrimitiveType'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Adds an input option for the underlying data source.", "class": "DataFrameReader" }, { "name": "options", "args": [ { "name": "options", "default": null, "annotation": null, "kind": "VAR_KEYWORD" } ], "doc_preview": "Adds input options for the underlying data source.", "class": "DataFrameReader" }, { "name": "orc", "args": [ { "name": "path", "default": null, "annotation": "PathOrPaths", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "mergeSchema", "default": "None", "annotation": "Optional[bool]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "pathGlobFilter", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "recursiveFileLookup", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "modifiedBefore", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "modifiedAfter", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Loads ORC files, returning the result as a :class:`DataFrame`.", "class": "DataFrameReader" }, { "name": "parquet", "args": [ { "name": "paths", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" }, { "name": "options", "default": null, "annotation": null, "kind": "VAR_KEYWORD" } ], "doc_preview": "Loads Parquet files, returning the result as a :class:`DataFrame`.", "class": "DataFrameReader" }, { "name": "schema", "args": [ { "name": "schema", "default": null, "annotation": "Union[StructType, str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Specifies the input schema.", "class": "DataFrameReader" }, { "name": "table", "args": [ { "name": "tableName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns the specified table as a :class:`DataFrame`.", "class": "DataFrameReader" }, { "name": "text", "args": [ { "name": "paths", "default": null, "annotation": "PathOrPaths", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "wholetext", "default": "False", "annotation": "bool", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "lineSep", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "pathGlobFilter", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "recursiveFileLookup", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "modifiedBefore", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "modifiedAfter", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Loads text files and returns a :class:`DataFrame` whose schema starts with a", "class": "DataFrameReader" } ], "DataFrameWriter": [ { "name": "bucketBy", "args": [ { "name": "numBuckets", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "", "class": "DataFrameWriter" }, { "name": "bucketBy", "args": [ { "name": "numBuckets", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col", "default": null, "annotation": "TupleOrListOfString", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "DataFrameWriter" }, { "name": "bucketBy", "args": [ { "name": "numBuckets", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "col", "default": null, "annotation": "Union[str, TupleOrListOfString]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Buckets the output by the given columns. If specified,", "class": "DataFrameWriter" }, { "name": "csv", "args": [ { "name": "path", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "mode", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "compression", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "sep", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "quote", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "escape", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "header", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "nullValue", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "escapeQuotes", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "quoteAll", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "dateFormat", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "timestampFormat", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "ignoreLeadingWhiteSpace", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "ignoreTrailingWhiteSpace", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "charToEscapeQuoteEscaping", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "encoding", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "emptyValue", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "lineSep", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Saves the content of the :class:`DataFrame` in CSV format at the specified path.", "class": "DataFrameWriter" }, { "name": "format", "args": [ { "name": "source", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Specifies the underlying output data source.", "class": "DataFrameWriter" }, { "name": "insertInto", "args": [ { "name": "tableName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "overwrite", "default": "None", "annotation": "Optional[bool]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Inserts the content of the :class:`DataFrame` to the specified table.", "class": "DataFrameWriter" }, { "name": "jdbc", "args": [ { "name": "url", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "table", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "mode", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "properties", "default": "None", "annotation": "Optional[Dict[str, str]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Saves the content of the :class:`DataFrame` to an external database table via JDBC.", "class": "DataFrameWriter" }, { "name": "json", "args": [ { "name": "path", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "mode", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "compression", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "dateFormat", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "timestampFormat", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "lineSep", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "encoding", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "ignoreNullFields", "default": "None", "annotation": "Optional[Union[bool, str]]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Saves the content of the :class:`DataFrame` in JSON format", "class": "DataFrameWriter" }, { "name": "mode", "args": [ { "name": "saveMode", "default": null, "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Specifies the behavior when data or table already exists.", "class": "DataFrameWriter" }, { "name": "option", "args": [ { "name": "key", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "value", "default": null, "annotation": "'OptionalPrimitiveType'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Adds an output option for the underlying data source.", "class": "DataFrameWriter" }, { "name": "options", "args": [ { "name": "options", "default": null, "annotation": null, "kind": "VAR_KEYWORD" } ], "doc_preview": "Adds output options for the underlying data source.", "class": "DataFrameWriter" }, { "name": "orc", "args": [ { "name": "path", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "mode", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "partitionBy", "default": "None", "annotation": "Optional[Union[str, List[str]]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "compression", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Saves the content of the :class:`DataFrame` in ORC format at the specified path.", "class": "DataFrameWriter" }, { "name": "parquet", "args": [ { "name": "path", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "mode", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "partitionBy", "default": "None", "annotation": "Optional[Union[str, List[str]]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "compression", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Saves the content of the :class:`DataFrame` in Parquet format at the specified path.", "class": "DataFrameWriter" }, { "name": "partitionBy", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "", "class": "DataFrameWriter" }, { "name": "partitionBy", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "", "class": "DataFrameWriter" }, { "name": "partitionBy", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Partitions the output by the given columns on the file system.", "class": "DataFrameWriter" }, { "name": "save", "args": [ { "name": "path", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "format", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "mode", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "partitionBy", "default": "None", "annotation": "Optional[Union[str, List[str]]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "options", "default": null, "annotation": null, "kind": "VAR_KEYWORD" } ], "doc_preview": "Saves the contents of the :class:`DataFrame` to a data source.", "class": "DataFrameWriter" }, { "name": "saveAsTable", "args": [ { "name": "name", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "format", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "mode", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "partitionBy", "default": "None", "annotation": "Optional[Union[str, List[str]]]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "options", "default": null, "annotation": null, "kind": "VAR_KEYWORD" } ], "doc_preview": "Saves the content of the :class:`DataFrame` as the specified table.", "class": "DataFrameWriter" }, { "name": "sortBy", "args": [ { "name": "col", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "", "class": "DataFrameWriter" }, { "name": "sortBy", "args": [ { "name": "col", "default": null, "annotation": "TupleOrListOfString", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "", "class": "DataFrameWriter" }, { "name": "sortBy", "args": [ { "name": "col", "default": null, "annotation": "Union[str, TupleOrListOfString]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Sorts the output in each bucket by the given columns on the file system.", "class": "DataFrameWriter" }, { "name": "text", "args": [ { "name": "path", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "compression", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "lineSep", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Saves the content of the DataFrame in a text file at the specified path.", "class": "DataFrameWriter" } ], "DataFrameWriterV2": [ { "name": "append", "args": [], "doc_preview": "Append the contents of the data frame to the output table.", "class": "DataFrameWriterV2" }, { "name": "create", "args": [], "doc_preview": "Create a new table from the contents of the data frame.", "class": "DataFrameWriterV2" }, { "name": "createOrReplace", "args": [], "doc_preview": "Create a new table or replace an existing table with the contents of the data frame.", "class": "DataFrameWriterV2" }, { "name": "option", "args": [ { "name": "key", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "value", "default": null, "annotation": "'OptionalPrimitiveType'", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Add a write option.", "class": "DataFrameWriterV2" }, { "name": "options", "args": [ { "name": "options", "default": null, "annotation": null, "kind": "VAR_KEYWORD" } ], "doc_preview": "Add write options.", "class": "DataFrameWriterV2" }, { "name": "overwrite", "args": [ { "name": "condition", "default": null, "annotation": "Column", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Overwrite rows matching the given filter condition with the contents of the data frame in", "class": "DataFrameWriterV2" }, { "name": "overwritePartitions", "args": [], "doc_preview": "Overwrite all partition for which the data frame contains at least one row with the contents", "class": "DataFrameWriterV2" }, { "name": "partitionedBy", "args": [ { "name": "col", "default": null, "annotation": "Column", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Partition the output table created by `create`, `createOrReplace`, or `replace` using", "class": "DataFrameWriterV2" }, { "name": "replace", "args": [], "doc_preview": "Replace an existing table with the contents of the data frame.", "class": "DataFrameWriterV2" }, { "name": "tableProperty", "args": [ { "name": "property", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "value", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Add table property.", "class": "DataFrameWriterV2" }, { "name": "using", "args": [ { "name": "provider", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Specifies a provider for the underlying output data source.", "class": "DataFrameWriterV2" } ], "Window": [ { "name": "orderBy", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Creates a :class:`WindowSpec` with the ordering defined.", "class": "Window" }, { "name": "partitionBy", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Creates a :class:`WindowSpec` with the partitioning defined.", "class": "Window" }, { "name": "rangeBetween", "args": [ { "name": "start", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "end", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Creates a :class:`WindowSpec` with the frame boundaries defined,", "class": "Window" }, { "name": "rowsBetween", "args": [ { "name": "start", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "end", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Creates a :class:`WindowSpec` with the frame boundaries defined,", "class": "Window" } ], "WindowSpec": [ { "name": "orderBy", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Defines the ordering columns in a :class:`WindowSpec`.", "class": "WindowSpec" }, { "name": "partitionBy", "args": [ { "name": "cols", "default": null, "annotation": null, "kind": "VAR_POSITIONAL" } ], "doc_preview": "Defines the partitioning columns in a :class:`WindowSpec`.", "class": "WindowSpec" }, { "name": "rangeBetween", "args": [ { "name": "start", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "end", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Defines the frame boundaries, from `start` (inclusive) to `end` (inclusive).", "class": "WindowSpec" }, { "name": "rowsBetween", "args": [ { "name": "start", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "end", "default": null, "annotation": "int", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Defines the frame boundaries, from `start` (inclusive) to `end` (inclusive).", "class": "WindowSpec" } ], "Catalog": [ { "name": "cacheTable", "args": [ { "name": "tableName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "storageLevel", "default": "None", "annotation": "Optional[StorageLevel]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Caches the specified table in-memory or with given storage level.", "class": "Catalog" }, { "name": "clearCache", "args": [], "doc_preview": "Removes all cached tables from the in-memory cache.", "class": "Catalog" }, { "name": "createExternalTable", "args": [ { "name": "tableName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "path", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "source", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "schema", "default": "None", "annotation": "Optional[StructType]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "options", "default": null, "annotation": null, "kind": "VAR_KEYWORD" } ], "doc_preview": "Creates a table based on the dataset in a data source.", "class": "Catalog" }, { "name": "createTable", "args": [ { "name": "tableName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "path", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "source", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "schema", "default": "None", "annotation": "Optional[StructType]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "description", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "options", "default": null, "annotation": null, "kind": "VAR_KEYWORD" } ], "doc_preview": "Creates a table based on the dataset in a data source.", "class": "Catalog" }, { "name": "currentCatalog", "args": [], "doc_preview": "Returns the current default catalog in this session.", "class": "Catalog" }, { "name": "currentDatabase", "args": [], "doc_preview": "Returns the current default database in this session.", "class": "Catalog" }, { "name": "databaseExists", "args": [ { "name": "dbName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Check if the database with the specified name exists.", "class": "Catalog" }, { "name": "dropGlobalTempView", "args": [ { "name": "viewName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Drops the global temporary view with the given view name in the catalog.", "class": "Catalog" }, { "name": "dropTempView", "args": [ { "name": "viewName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Drops the local temporary view with the given view name in the catalog.", "class": "Catalog" }, { "name": "functionExists", "args": [ { "name": "functionName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "dbName", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Check if the function with the specified name exists.", "class": "Catalog" }, { "name": "getDatabase", "args": [ { "name": "dbName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Get the database with the specified name.", "class": "Catalog" }, { "name": "getFunction", "args": [ { "name": "functionName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Get the function with the specified name. This function can be a temporary function or a", "class": "Catalog" }, { "name": "getTable", "args": [ { "name": "tableName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Get the table or view with the specified name. This table can be a temporary view or a", "class": "Catalog" }, { "name": "isCached", "args": [ { "name": "tableName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns true if the table is currently cached in-memory.", "class": "Catalog" }, { "name": "listCatalogs", "args": [ { "name": "pattern", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a list of catalogs in this session.", "class": "Catalog" }, { "name": "listColumns", "args": [ { "name": "tableName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "dbName", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a list of columns for the given table/view in the specified database.", "class": "Catalog" }, { "name": "listDatabases", "args": [ { "name": "pattern", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a list of databases available across all sessions.", "class": "Catalog" }, { "name": "listFunctions", "args": [ { "name": "dbName", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "pattern", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a list of functions registered in the specified database.", "class": "Catalog" }, { "name": "listTables", "args": [ { "name": "dbName", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "pattern", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Returns a list of tables/views in the specified database.", "class": "Catalog" }, { "name": "recoverPartitions", "args": [ { "name": "tableName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Recovers all the partitions of the given table and updates the catalog.", "class": "Catalog" }, { "name": "refreshByPath", "args": [ { "name": "path", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Invalidates and refreshes all the cached data (and the associated metadata) for any", "class": "Catalog" }, { "name": "refreshTable", "args": [ { "name": "tableName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Invalidates and refreshes all the cached data and metadata of the given table.", "class": "Catalog" }, { "name": "registerFunction", "args": [ { "name": "name", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "f", "default": null, "annotation": "Callable[..., Any]", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "returnType", "default": "None", "annotation": "Optional['DataType']", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "An alias for :func:`spark.udf.register`.", "class": "Catalog" }, { "name": "setCurrentCatalog", "args": [ { "name": "catalogName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Sets the current default catalog in this session.", "class": "Catalog" }, { "name": "setCurrentDatabase", "args": [ { "name": "dbName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Sets the current default database in this session.", "class": "Catalog" }, { "name": "tableExists", "args": [ { "name": "tableName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" }, { "name": "dbName", "default": "None", "annotation": "Optional[str]", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Check if the table or view with the specified name exists.", "class": "Catalog" }, { "name": "uncacheTable", "args": [ { "name": "tableName", "default": null, "annotation": "str", "kind": "POSITIONAL_OR_KEYWORD" } ], "doc_preview": "Removes the specified table from the in-memory cache.", "class": "Catalog" } ] }