From 8bd5e35543bcfc6ec628611700261ea9406a6495 Mon Sep 17 00:00:00 2001
From: "Carlos E. Velasco" <33700617+cvelac4@users.noreply.github.com>
Date: Tue, 25 Feb 2025 22:10:46 -0800
Subject: [PATCH 1/5] Created using Colab. Using PySpark to show data from the
 .csv file.

---
 PySpark.ipynb | 300 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 300 insertions(+)
 create mode 100644 PySpark.ipynb

diff --git a/PySpark.ipynb b/PySpark.ipynb
new file mode 100644
index 0000000..8400db7
--- /dev/null
+++ b/PySpark.ipynb
@@ -0,0 +1,300 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "authorship_tag": "ABX9TyMGUy7O01Iif+5ned4ITEW2",
+      "include_colab_link": true
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "view-in-github",
+        "colab_type": "text"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/github/cvelac4/Algorithm-and-Leetcode/blob/master/PySpark.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Title and Explenation"
+      ],
+      "metadata": {
+        "id": "sGOX0xNhmvIN"
+      }
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 1,
+      "metadata": {
+        "id": "N6iVXJp7jQME"
+      },
+      "outputs": [],
+      "source": [
+        "# Predictions of Suporstore data using Advanced ML PySpark.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Download Pyspark"
+      ],
+      "metadata": {
+        "id": "sJDMbsjkmhJN"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "!pip install PySpark"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "DFAon71alA3W",
+        "outputId": "d85f7c64-1165-4048-9a15-ee1da3cf1969"
+      },
+      "execution_count": 2,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Requirement already satisfied: PySpark in /usr/local/lib/python3.11/dist-packages (3.5.4)\n",
+            "Requirement already satisfied: py4j==0.10.9.7 in /usr/local/lib/python3.11/dist-packages (from PySpark) (0.10.9.7)\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Create a Session"
+      ],
+      "metadata": {
+        "id": "BvzJXcs7m5JW"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from pyspark.sql import SparkSession\n",
+        "\n",
+        "spark = SparkSession.builder.appName('SalesForecasting').getOrCreate()"
+      ],
+      "metadata": {
+        "id": "bMpAL_cPmfH8"
+      },
+      "execution_count": 3,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "#Load and Explore the Data\n"
+      ],
+      "metadata": {
+        "id": "hcGpdLwapX6h"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "#data\n",
+        "path = '/content/1740463998446_a2368e63c8e87f60.csv'\n",
+        "df = spark.read.csv(path, header=True, inferSchema=True )\n",
+        "\n",
+        "#Display the Schema\n",
+        "df.printSchema()\n",
+        "\n",
+        "#View the sample data\n",
+        "df.show()\n"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "m_uZfJ20pfGn",
+        "outputId": "f5911b7c-275f-42d5-9072-f0dd910b0dfa"
+      },
+      "execution_count": 5,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "root\n",
+            " |-- ID: integer (nullable = true)\n",
+            " |-- Order_id: string (nullable = true)\n",
+            " |-- Order_Date: string (nullable = true)\n",
+            " |-- Ship _Date: date (nullable = true)\n",
+            " |-- Ship_Mode: string (nullable = true)\n",
+            " |-- Customer_id: string (nullable = true)\n",
+            " |-- Customer_Name: string (nullable = true)\n",
+            " |-- Segment: string (nullable = true)\n",
+            " |-- Country: string (nullable = true)\n",
+            " |-- City: string (nullable = true)\n",
+            " |-- State: string (nullable = true)\n",
+            " |-- Postal_Code: integer (nullable = true)\n",
+            " |-- Region: string (nullable = true)\n",
+            " |-- Product_ ID: string (nullable = true)\n",
+            " |-- Category: string (nullable = true)\n",
+            " |-- Sub_Category: string (nullable = true)\n",
+            " |-- Product_Name: string (nullable = true)\n",
+            " |-- Sales: string (nullable = true)\n",
+            " |-- Quantity: string (nullable = true)\n",
+            " |-- Discount: string (nullable = true)\n",
+            " |-- Profit: double (nullable = true)\n",
+            " |-- user_id: double (nullable = true)\n",
+            " |-- state_id: double (nullable = true)\n",
+            " |-- order_s: string (nullable = true)\n",
+            "\n",
+            "+---+--------------+----------+----------+--------------+-----------+------------------+-----------+-------------+---------------+--------------+-----------+-------+---------------+---------------+------------+--------------------+--------+--------+--------+--------+-------+--------+-------+\n",
+            "| ID|      Order_id|Order_Date|Ship _Date|     Ship_Mode|Customer_id|     Customer_Name|    Segment|      Country|           City|         State|Postal_Code| Region|    Product_ ID|       Category|Sub_Category|        Product_Name|   Sales|Quantity|Discount|  Profit|user_id|state_id|order_s|\n",
+            "+---+--------------+----------+----------+--------------+-----------+------------------+-----------+-------------+---------------+--------------+-----------+-------+---------------+---------------+------------+--------------------+--------+--------+--------+--------+-------+--------+-------+\n",
+            "|  1|CA-2023-152156|2023-11-08|2023-11-11|  Second Class|   CG-12520|       Claire Gute|   Consumer|United States|      Henderson|      Kentucky|      42420|  South|FUR-BO-10001798|      Furniture|   Bookcases|Bush Somerset Col...|  261.96|       2|       0| 41.9136|   NULL|    NULL|   NULL|\n",
+            "|  2|CA-2023-152156|2023-11-08|2023-11-11|  Second Class|   CG-12520|       Claire Gute|   Consumer|United States|      Henderson|      Kentucky|      42420|  South|FUR-CH-10000454|      Furniture|      Chairs|Hon Deluxe Fabric...|  731.94|       3|       0| 219.582|    1.0|    NULL|   NULL|\n",
+            "|  3|CA-2023-138688|2023-06-12|2023-06-16|  Second Class|   DV-13045|   Darrin Van Huff|  Corporate|United States|    Los Angeles|    California|      90036|   West|OFF-LA-10000240|Office Supplies|      Labels|Self-Adhesive Add...|   14.62|       2|       0|  6.8714|   NULL|    NULL|   NULL|\n",
+            "|  4|US-2022-108966|2022-10-11|2022-10-18|Standard Class|   SO-20335|    Sean O'Donnell|   Consumer|United States|Fort Lauderdale|       Florida|      33311|  South|FUR-TA-10000577|      Furniture|      Tables|Bretford CR4500 S...|957.5775|       5|    0.45|-383.031|   NULL|    NULL|   NULL|\n",
+            "|  5|US-2022-108966|2022-10-11|2022-10-18|Standard Class|   SO-20335|    Sean O'Donnell|   Consumer|United States|Fort Lauderdale|       Florida|      33311|  South|OFF-ST-10000760|Office Supplies|     Storage|Eldon Fold 'N Rol...|  22.368|       2|     0.2|  2.5164|   NULL|    NULL|   NULL|\n",
+            "|  6|CA-2021-115812|2021-06-09|2021-06-14|Standard Class|   BH-11710|   Brosina Hoffman|   Consumer|United States|    Los Angeles|    California|      90032|   West|FUR-FU-10001487|      Furniture| Furnishings|Eldon Expressions...|   48.86|       7|       0| 14.1694|   NULL|    NULL|   NULL|\n",
+            "|  7|CA-2021-115812|2021-06-09|2021-06-14|Standard Class|   BH-11710|   Brosina Hoffman|   Consumer|United States|    Los Angeles|    California|      90032|   West|OFF-AR-10002833|Office Supplies|         Art|          Newell 322|    7.28|       4|       0|  1.9656|   NULL|     1.0|   NULL|\n",
+            "|  8|CA-2021-115812|2021-06-09|2021-06-14|Standard Class|   BH-11710|   Brosina Hoffman|   Consumer|United States|    Los Angeles|    California|      90032|   West|TEC-PH-10002275|     Technology|      Phones|Mitel 5320 IP Pho...| 907.152|       6|     0.2| 90.7152|   NULL|    NULL|   NULL|\n",
+            "|  9|CA-2021-115812|2021-06-09|2021-06-14|Standard Class|   BH-11710|   Brosina Hoffman|   Consumer|United States|    Los Angeles|    California|      90032|   West|OFF-BI-10003910|Office Supplies|     Binders|DXL Angle-View Bi...|  18.504|       3|     0.2|  5.7825|   NULL|     2.0|   NULL|\n",
+            "| 10|CA-2021-115812|2021-06-09|2021-06-14|Standard Class|   BH-11710|   Brosina Hoffman|   Consumer|United States|    Los Angeles|    California|      90032|   West|OFF-AP-10002892|Office Supplies|  Appliances|Belkin F5C206VTEL...|   114.9|       5|       0|   34.47|   NULL|    NULL|   NULL|\n",
+            "| 11|CA-2021-115812|2021-06-09|2021-06-14|Standard Class|   BH-11710|   Brosina Hoffman|   Consumer|United States|    Los Angeles|    California|      90032|   West|FUR-TA-10001539|      Furniture|      Tables|Chromcraft Rectan...|1706.184|       9|     0.2| 85.3092|   NULL|    NULL|   NULL|\n",
+            "| 12|CA-2021-115812|2021-06-09|2021-06-14|Standard Class|   BH-11710|   Brosina Hoffman|   Consumer|United States|    Los Angeles|    California|      90032|   West|TEC-PH-10002033|     Technology|      Phones|Konftel 250 Confe...| 911.424|       4|     0.2| 68.3568|   NULL|    NULL|   NULL|\n",
+            "| 13|CA-2024-114412|2024-04-15|2024-04-20|Standard Class|   AA-10480|      Andrew Allen|   Consumer|United States|        Concord|North Carolina|      28027|  South|OFF-PA-10002365|Office Supplies|       Paper|          Xerox 1967|  15.552|       3|     0.2|  5.4432|   NULL|    NULL|      s|\n",
+            "| 14|CA-2023-161389|2023-12-05|2023-12-10|Standard Class|   IM-15070|      Irene Maddox|   Consumer|United States|        Seattle|    Washington|      98103|   West|OFF-BI-10003656|Office Supplies|     Binders|Fellowes PB200 Pl...| 407.976|       3|     0.2|132.5922|   NULL|    NULL|   NULL|\n",
+            "| 15|US-2022-118983|2022-11-22|2022-11-26|Standard Class|   HP-14815|     Harold Pawlan|Home Office|United States|     Fort Worth|         Texas|      76106|Central|OFF-AP-10002311|Office Supplies|  Appliances|Holmes Replacemen...|   68.81|       5|     0.8|-123.858|   NULL|    NULL|   NULL|\n",
+            "| 16|US-2022-118983|2022-11-22|2022-11-26|Standard Class|   HP-14815|     Harold Pawlan|Home Office|United States|     Fort Worth|         Texas|      76106|Central|OFF-BI-10000756|Office Supplies|     Binders|Storex DuraTech R...|   2.544|       3|     0.8|  -3.816|   NULL|    NULL|   NULL|\n",
+            "| 17|CA-2021-105893|2021-11-11|2021-11-18|Standard Class|   PK-19075|         Pete Kriz|   Consumer|United States|        Madison|     Wisconsin|      53711|Central|OFF-ST-10004186|Office Supplies|     Storage|\"Stur-D-Stor Shel...|  665.88|       6|       0| 13.3176|   NULL|    NULL|   NULL|\n",
+            "| 18|CA-2021-167164|2021-05-13|2021-05-15|  Second Class|   AG-10270|   Alejandro Grove|   Consumer|United States|    West Jordan|          Utah|      84084|   West|OFF-ST-10000107|Office Supplies|     Storage|Fellowes Super St...|    55.5|       2|       0|    9.99|   NULL|    NULL|      s|\n",
+            "| 19|CA-2021-143336|2021-08-27|2021-09-01|  Second Class|   ZD-21925|Zuschuss Donatelli|   Consumer|United States|  San Francisco|    California|      94109|   West|OFF-AR-10003056|Office Supplies|         Art|          Newell 341|    8.56|       2|       0|  2.4824|   NULL|    NULL|   NULL|\n",
+            "| 20|CA-2021-143336|2021-08-27|2021-09-01|  Second Class|   ZD-21925|Zuschuss Donatelli|   Consumer|United States|  San Francisco|    California|      94109|   West|TEC-PH-10001949|     Technology|      Phones|Cisco SPA 501G IP...|  213.48|       3|     0.2|  16.011|   NULL|    NULL|   NULL|\n",
+            "+---+--------------+----------+----------+--------------+-----------+------------------+-----------+-------------+---------------+--------------+-----------+-------+---------------+---------------+------------+--------------------+--------+--------+--------+--------+-------+--------+-------+\n",
+            "only showing top 20 rows\n",
+            "\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "#Data Processing\n",
+        "\n",
+        "\n",
+        "\n",
+        "*   Convert Data\n",
+        "*   Handdle Missing\n",
+        "Aggregate on daily level\n",
+        "\n",
+        "\n",
+        "\n",
+        "\n",
+        "\n"
+      ],
+      "metadata": {
+        "id": "V9V9fdGZvSl4"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "#convert Order_data to data type\n",
+        "from pyspark.sql.functions import to_date, col,sum, dayofmonth, month, year,lag\n",
+        "\n",
+        "df = df.withColumn('Order_Date', col('Order_Date').cast('date') )\n",
+        "\n",
+        "df.printSchema()\n",
+        "\n",
+        "#Aggregate\n",
+        "daily_sales = df.groupBy('Order_Date').agg(sum('Sales').alias('Daily_Sales'))\n",
+        "\n",
+        "daily_sales.show()"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "McHKdq5sv-VT",
+        "outputId": "645d537b-68c9-4527-dbe5-95d7b6cbca4d"
+      },
+      "execution_count": 8,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "root\n",
+            " |-- ID: integer (nullable = true)\n",
+            " |-- Order_id: string (nullable = true)\n",
+            " |-- Order_Date: date (nullable = true)\n",
+            " |-- Ship _Date: date (nullable = true)\n",
+            " |-- Ship_Mode: string (nullable = true)\n",
+            " |-- Customer_id: string (nullable = true)\n",
+            " |-- Customer_Name: string (nullable = true)\n",
+            " |-- Segment: string (nullable = true)\n",
+            " |-- Country: string (nullable = true)\n",
+            " |-- City: string (nullable = true)\n",
+            " |-- State: string (nullable = true)\n",
+            " |-- Postal_Code: integer (nullable = true)\n",
+            " |-- Region: string (nullable = true)\n",
+            " |-- Product_ ID: string (nullable = true)\n",
+            " |-- Category: string (nullable = true)\n",
+            " |-- Sub_Category: string (nullable = true)\n",
+            " |-- Product_Name: string (nullable = true)\n",
+            " |-- Sales: string (nullable = true)\n",
+            " |-- Quantity: string (nullable = true)\n",
+            " |-- Discount: string (nullable = true)\n",
+            " |-- Profit: double (nullable = true)\n",
+            " |-- user_id: double (nullable = true)\n",
+            " |-- state_id: double (nullable = true)\n",
+            " |-- order_s: string (nullable = true)\n",
+            "\n",
+            "+----------+------------------+\n",
+            "|Order_Date|       Daily_Sales|\n",
+            "+----------+------------------+\n",
+            "|2021-08-27|           2070.13|\n",
+            "|2024-09-18|1454.7299999999998|\n",
+            "|2021-06-22|          1975.498|\n",
+            "|2022-03-28|           243.344|\n",
+            "|2022-07-31|          3712.162|\n",
+            "|2023-07-15|             380.2|\n",
+            "|2021-10-11|          1381.164|\n",
+            "|2021-01-27|            426.67|\n",
+            "|2023-11-08| 993.9000000000001|\n",
+            "|2024-08-27|5992.0779999999995|\n",
+            "|2022-11-29|2760.1680000000006|\n",
+            "|2024-06-04|           279.414|\n",
+            "|2024-06-12|          1679.968|\n",
+            "|2021-11-25| 4415.695000000001|\n",
+            "|2022-12-25| 4204.968000000001|\n",
+            "|2023-05-22|1799.2000000000003|\n",
+            "|2023-09-14|          1137.338|\n",
+            "|2021-10-02|           588.736|\n",
+            "|2022-08-02|          1290.478|\n",
+            "|2022-07-27|             29.97|\n",
+            "+----------+------------------+\n",
+            "only showing top 20 rows\n",
+            "\n"
+          ]
+        }
+      ]
+    }
+  ]
+}
\ No newline at end of file

From 34ada04b4983362fe4eb530b2d4bc1bc3911edc0 Mon Sep 17 00:00:00 2001
From: "Carlos E. Velasco" <33700617+cvelac4@users.noreply.github.com>
Date: Mon, 3 Mar 2025 21:41:03 -0800
Subject: [PATCH 2/5] Created using Colab

---
 PySpark.ipynb | 42 ++++++++++++++++++++++++++++++++++++------
 1 file changed, 36 insertions(+), 6 deletions(-)

diff --git a/PySpark.ipynb b/PySpark.ipynb
index 8400db7..8a12d8d 100644
--- a/PySpark.ipynb
+++ b/PySpark.ipynb
@@ -4,7 +4,7 @@
   "metadata": {
     "colab": {
       "provenance": [],
-      "authorship_tag": "ABX9TyMGUy7O01Iif+5ned4ITEW2",
+      "authorship_tag": "ABX9TyP/PsSJyKDfLHGFwZVAJl79",
       "include_colab_link": true
     },
     "kernelspec": {
@@ -37,7 +37,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 1,
+      "execution_count": null,
       "metadata": {
         "id": "N6iVXJp7jQME"
       },
@@ -67,7 +67,7 @@
         "id": "DFAon71alA3W",
         "outputId": "d85f7c64-1165-4048-9a15-ee1da3cf1969"
       },
-      "execution_count": 2,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -98,7 +98,7 @@
       "metadata": {
         "id": "bMpAL_cPmfH8"
       },
-      "execution_count": 3,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -130,7 +130,7 @@
         "id": "m_uZfJ20pfGn",
         "outputId": "f5911b7c-275f-42d5-9072-f0dd910b0dfa"
       },
-      "execution_count": 5,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -234,7 +234,7 @@
         "id": "McHKdq5sv-VT",
         "outputId": "645d537b-68c9-4527-dbe5-95d7b6cbca4d"
       },
-      "execution_count": 8,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -295,6 +295,36 @@
           ]
         }
       ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "daily_sales.show(10)\n",
+        "daily_sales = daily_sales.sort('Order_Date')\n",
+        "daily_sales.show(10)"
+      ],
+      "metadata": {
+        "id": "N0Fqk2wHNFUB",
+        "outputId": "91b1cc0a-8547-4a22-d75f-e59c9cf72e31",
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 176
+        }
+      },
+      "execution_count": 1,
+      "outputs": [
+        {
+          "output_type": "error",
+          "ename": "NameError",
+          "evalue": "name 'daily_sales' is not defined",
+          "traceback": [
+            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
+            "\u001b[0;32m<ipython-input-1-9ccd588ae7a1>\u001b[0m in \u001b[0;36m<cell line: 0>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdaily_sales\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mdaily_sales\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdaily_sales\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Order_Date'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mdaily_sales\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+            "\u001b[0;31mNameError\u001b[0m: name 'daily_sales' is not defined"
+          ]
+        }
+      ]
     }
   ]
 }
\ No newline at end of file

From 9c08dd2304a0fe4f45b2a3bc35225c2b14e43408 Mon Sep 17 00:00:00 2001
From: "Carlos E. Velasco" <33700617+cvelac4@users.noreply.github.com>
Date: Mon, 3 Mar 2025 21:56:54 -0800
Subject: [PATCH 3/5] Created using Colab

---
 PySpark.ipynb | 72 +++++++++++++++++++++++++++++++++++----------------
 1 file changed, 50 insertions(+), 22 deletions(-)

diff --git a/PySpark.ipynb b/PySpark.ipynb
index 8a12d8d..5d344fe 100644
--- a/PySpark.ipynb
+++ b/PySpark.ipynb
@@ -4,7 +4,7 @@
   "metadata": {
     "colab": {
       "provenance": [],
-      "authorship_tag": "ABX9TyP/PsSJyKDfLHGFwZVAJl79",
+      "authorship_tag": "ABX9TyNxAkUJklJhGRAUU039mAKM",
       "include_colab_link": true
     },
     "kernelspec": {
@@ -37,7 +37,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 19,
       "metadata": {
         "id": "N6iVXJp7jQME"
       },
@@ -65,15 +65,15 @@
           "base_uri": "https://localhost:8080/"
         },
         "id": "DFAon71alA3W",
-        "outputId": "d85f7c64-1165-4048-9a15-ee1da3cf1969"
+        "outputId": "f4104f2a-2fcb-4898-da2e-ecacac21bdf1"
       },
-      "execution_count": null,
+      "execution_count": 22,
       "outputs": [
         {
           "output_type": "stream",
           "name": "stdout",
           "text": [
-            "Requirement already satisfied: PySpark in /usr/local/lib/python3.11/dist-packages (3.5.4)\n",
+            "Requirement already satisfied: PySpark in /usr/local/lib/python3.11/dist-packages (3.5.5)\n",
             "Requirement already satisfied: py4j==0.10.9.7 in /usr/local/lib/python3.11/dist-packages (from PySpark) (0.10.9.7)\n"
           ]
         }
@@ -98,7 +98,7 @@
       "metadata": {
         "id": "bMpAL_cPmfH8"
       },
-      "execution_count": null,
+      "execution_count": 24,
       "outputs": []
     },
     {
@@ -128,9 +128,9 @@
           "base_uri": "https://localhost:8080/"
         },
         "id": "m_uZfJ20pfGn",
-        "outputId": "f5911b7c-275f-42d5-9072-f0dd910b0dfa"
+        "outputId": "8f0000de-e0d7-48fa-9ec3-7aebc2aa348f"
       },
-      "execution_count": null,
+      "execution_count": 25,
       "outputs": [
         {
           "output_type": "stream",
@@ -232,9 +232,9 @@
           "base_uri": "https://localhost:8080/"
         },
         "id": "McHKdq5sv-VT",
-        "outputId": "645d537b-68c9-4527-dbe5-95d7b6cbca4d"
+        "outputId": "8c4ed271-bdc8-4f37-b422-d803f453d4a0"
       },
-      "execution_count": null,
+      "execution_count": 26,
       "outputs": [
         {
           "output_type": "stream",
@@ -300,28 +300,56 @@
       "cell_type": "code",
       "source": [
         "daily_sales.show(10)\n",
+        "\n",
+        "#Sort the date.\n",
         "daily_sales = daily_sales.sort('Order_Date')\n",
         "daily_sales.show(10)"
       ],
       "metadata": {
         "id": "N0Fqk2wHNFUB",
-        "outputId": "91b1cc0a-8547-4a22-d75f-e59c9cf72e31",
+        "outputId": "9a4423e6-f3e6-4cc1-dbe0-1ae070f14fa1",
         "colab": {
-          "base_uri": "https://localhost:8080/",
-          "height": 176
+          "base_uri": "https://localhost:8080/"
         }
       },
-      "execution_count": 1,
+      "execution_count": 27,
       "outputs": [
         {
-          "output_type": "error",
-          "ename": "NameError",
-          "evalue": "name 'daily_sales' is not defined",
-          "traceback": [
-            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
-            "\u001b[0;32m<ipython-input-1-9ccd588ae7a1>\u001b[0m in \u001b[0;36m<cell line: 0>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdaily_sales\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mdaily_sales\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdaily_sales\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Order_Date'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mdaily_sales\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-            "\u001b[0;31mNameError\u001b[0m: name 'daily_sales' is not defined"
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "+----------+------------------+\n",
+            "|Order_Date|       Daily_Sales|\n",
+            "+----------+------------------+\n",
+            "|2021-08-27|           2070.13|\n",
+            "|2024-09-18|1454.7299999999998|\n",
+            "|2021-06-22|          1975.498|\n",
+            "|2022-03-28|           243.344|\n",
+            "|2022-07-31|          3712.162|\n",
+            "|2023-07-15|             380.2|\n",
+            "|2021-10-11|          1381.164|\n",
+            "|2021-01-27|            426.67|\n",
+            "|2023-11-08| 993.9000000000001|\n",
+            "|2024-08-27|5992.0779999999995|\n",
+            "+----------+------------------+\n",
+            "only showing top 10 rows\n",
+            "\n",
+            "+----------+-----------------+\n",
+            "|Order_Date|      Daily_Sales|\n",
+            "+----------+-----------------+\n",
+            "|      NULL|          111.104|\n",
+            "|2021-01-03|             NULL|\n",
+            "|2021-01-04|           288.06|\n",
+            "|2021-01-05|           19.536|\n",
+            "|2021-01-06|           4407.1|\n",
+            "|2021-01-07|87.15799999999999|\n",
+            "|2021-01-09|           40.544|\n",
+            "|2021-01-10|            54.83|\n",
+            "|2021-01-11|             9.94|\n",
+            "|2021-01-13|         3553.795|\n",
+            "+----------+-----------------+\n",
+            "only showing top 10 rows\n",
+            "\n"
           ]
         }
       ]

From ba71a29cae30dde83092371dc735710b93ce6be8 Mon Sep 17 00:00:00 2001
From: "Carlos E. Velasco" <33700617+cvelac4@users.noreply.github.com>
Date: Mon, 3 Mar 2025 22:28:06 -0800
Subject: [PATCH 4/5] Created using Colab

---
 PySpark.ipynb | 131 +++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 130 insertions(+), 1 deletion(-)

diff --git a/PySpark.ipynb b/PySpark.ipynb
index 5d344fe..3c2b1d2 100644
--- a/PySpark.ipynb
+++ b/PySpark.ipynb
@@ -4,7 +4,7 @@
   "metadata": {
     "colab": {
       "provenance": [],
-      "authorship_tag": "ABX9TyNxAkUJklJhGRAUU039mAKM",
+      "authorship_tag": "ABX9TyO+U61SsUd5D9aYKQLGAMPF",
       "include_colab_link": true
     },
     "kernelspec": {
@@ -353,6 +353,135 @@
           ]
         }
       ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "#Featured Engineering"
+      ],
+      "metadata": {
+        "id": "Cb2BpKZUuy6Q"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from pyspark.sql.functions import dayofmonth, month, year, lag\n",
+        "from pyspark.sql.window import Window\n",
+        "\n",
+        "#Window\n",
+        "wind_spec = Window.orderBy('Order_Date')\n",
+        "\n",
+        "#Add Lag (previous day sales)\n",
+        "daily_sales = daily_sales.withColumn('Prev_Day_Sales', lag('Daily_Sales').over(wind_spec))\n",
+        "\n",
+        "daily_sales = daily_sales.withColumn('Day', dayofmonth(col('Order_Date')))\n",
+        "daily_sales = daily_sales.withColumn('Month', month(col('Order_Date')))\n",
+        "daily_sales = daily_sales.withColumn('Year', year(col('Order_Date')))\n",
+        "\n",
+        "#Drop NA values\n",
+        "daily_sales = daily_sales.na.drop()\n",
+        "\n",
+        "daily_sales.show(10)"
+      ],
+      "metadata": {
+        "id": "_TvbaBNVu50q",
+        "outputId": "6440540e-b667-46c3-9179-f185a70040e2",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
+      },
+      "execution_count": 30,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "+----------+-----------------+-----------------+---+-----+----+\n",
+            "|Order_Date|      Daily_Sales|   Prev_Day_Sales|Day|Month|Year|\n",
+            "+----------+-----------------+-----------------+---+-----+----+\n",
+            "|2021-01-05|           19.536|           288.06|  5|    1|2021|\n",
+            "|2021-01-06|           4407.1|           19.536|  6|    1|2021|\n",
+            "|2021-01-07|87.15799999999999|           4407.1|  7|    1|2021|\n",
+            "|2021-01-09|           40.544|87.15799999999999|  9|    1|2021|\n",
+            "|2021-01-10|            54.83|           40.544| 10|    1|2021|\n",
+            "|2021-01-11|             9.94|            54.83| 11|    1|2021|\n",
+            "|2021-01-13|         3553.795|             9.94| 13|    1|2021|\n",
+            "|2021-01-14|            61.96|         3553.795| 14|    1|2021|\n",
+            "|2021-01-15|           149.95|            61.96| 15|    1|2021|\n",
+            "|2021-01-16|          299.964|           149.95| 16|    1|2021|\n",
+            "+----------+-----------------+-----------------+---+-----+----+\n",
+            "only showing top 10 rows\n",
+            "\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "#Training and Testing"
+      ],
+      "metadata": {
+        "id": "gvQchto0yyZX"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from pyspark.ml.feature import VectorAssembler\n",
+        "\n",
+        "#Assemble the fetures into singal vector\n",
+        "feature_col = ['Day', 'Month', 'Year', 'Prev_Day_Sales']\n",
+        "assembler = VectorAssembler(inputCols = feature_col, outputCol = 'features')\n",
+        "\n",
+        "daily_sales_update = assembler.transform(daily_sales).select('features', 'Daily_Sales')\n",
+        "\n",
+        "daily_sales_update.show(10)"
+      ],
+      "metadata": {
+        "id": "wSLIMTLjy3pl",
+        "outputId": "ec1d4068-8faf-4097-b49b-3979149cc836",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
+      },
+      "execution_count": 31,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "+--------------------+-----------------+\n",
+            "|            features|      Daily_Sales|\n",
+            "+--------------------+-----------------+\n",
+            "|[5.0,1.0,2021.0,2...|           19.536|\n",
+            "|[6.0,1.0,2021.0,1...|           4407.1|\n",
+            "|[7.0,1.0,2021.0,4...|87.15799999999999|\n",
+            "|[9.0,1.0,2021.0,8...|           40.544|\n",
+            "|[10.0,1.0,2021.0,...|            54.83|\n",
+            "|[11.0,1.0,2021.0,...|             9.94|\n",
+            "|[13.0,1.0,2021.0,...|         3553.795|\n",
+            "|[14.0,1.0,2021.0,...|            61.96|\n",
+            "|[15.0,1.0,2021.0,...|           149.95|\n",
+            "|[16.0,1.0,2021.0,...|          299.964|\n",
+            "+--------------------+-----------------+\n",
+            "only showing top 10 rows\n",
+            "\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "train, test = daily_sales_update.randomSplit([0.8, 0.2], seed= 42)"
+      ],
+      "metadata": {
+        "id": "cVHQJ5Er06Pl"
+      },
+      "execution_count": 32,
+      "outputs": []
     }
   ]
 }
\ No newline at end of file

From 0cb5a6f1dca1545177d0fed79df18f4c7ab0a2ea Mon Sep 17 00:00:00 2001
From: "Carlos E. Velasco" <33700617+cvelac4@users.noreply.github.com>
Date: Mon, 3 Mar 2025 23:10:40 -0800
Subject: [PATCH 5/5] Created using Colab

---
 PySpark.ipynb | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/PySpark.ipynb b/PySpark.ipynb
index 3c2b1d2..ff571e1 100644
--- a/PySpark.ipynb
+++ b/PySpark.ipynb
@@ -385,11 +385,11 @@
         "daily_sales.show(10)"
       ],
       "metadata": {
-        "id": "_TvbaBNVu50q",
-        "outputId": "6440540e-b667-46c3-9179-f185a70040e2",
         "colab": {
           "base_uri": "https://localhost:8080/"
-        }
+        },
+        "id": "_TvbaBNVu50q",
+        "outputId": "6440540e-b667-46c3-9179-f185a70040e2"
       },
       "execution_count": 30,
       "outputs": [
@@ -440,11 +440,11 @@
         "daily_sales_update.show(10)"
       ],
       "metadata": {
-        "id": "wSLIMTLjy3pl",
-        "outputId": "ec1d4068-8faf-4097-b49b-3979149cc836",
         "colab": {
           "base_uri": "https://localhost:8080/"
-        }
+        },
+        "id": "wSLIMTLjy3pl",
+        "outputId": "ec1d4068-8faf-4097-b49b-3979149cc836"
       },
       "execution_count": 31,
       "outputs": [
