|
106 | 106 | "from pyspark.sql import SparkSession\n", |
107 | 107 | "\n", |
108 | 108 | "# METHOD I:\n", |
109 | | - "# Update Spark configurations of the following two extraClassPath prior to initiating a Spark session:\n", |
| 109 | + "# Update Spark configurations of the following two extraClassPath with the JDBC driver location\n", |
| 110 | + "# prior to initiating a Spark session:\n", |
110 | 111 | "# spark.driver.extraClassPath\n", |
111 | 112 | "# spark.executor.extraClassPath\n", |
112 | 113 | "#\n", |
|
122 | 123 | "\n", |
123 | 124 | "\"\"\"\n", |
124 | 125 | "# METHOD II:\n", |
125 | | - "# Use \"PYSPARK_SUBMIT_ARGS\" with \"--packages\" option.\n", |
| 126 | + "# Use \"PYSPARK_SUBMIT_ARGS\" with \"--packages\" option. (The preferred way)\n", |
126 | 127 | "# \n", |
| 128 | + "# Usage:\n", |
| 129 | + "# os.environ[\"PYSPARK_SUBMIT_ARGS\"] = \"--packages dialect:dialect-specific-jdbc-connector:version# pyspark-shell\"\n", |
| 130 | + "#\n", |
127 | 131 | "# NOTE:\n", |
128 | 132 | "# If you don't connnect to mysql, replace the mysql's connector by the other database's JDBC connector \n", |
129 | 133 | "# in the following line.\n", |
130 | 134 | "\"\"\"\n", |
| 135 | + "# Set PYSPARK_SUBMIT_ARGS\n", |
131 | 136 | "#os.environ[\"PYSPARK_SUBMIT_ARGS\"] = \"--packages mysql:mysql-connector-java:5.1.39 pyspark-shell\"\n", |
132 | 137 | "\n", |
133 | 138 | "\"\"\"\n", |
|
137 | 142 | "# spark.executor.extraClassPath\n", |
138 | 143 | "\"\"\" \n", |
139 | 144 | "# Initiate a Spark Session\n", |
140 | | - "#spark = SparkSession.builder.\\\n", |
141 | | - "# appName(\"Spark JDBC to Databases - ipynb\").getOrCreate()" |
| 145 | + "#spark = SparkSession.builder.appName(\"Spark JDBC to Databases - ipynb\").getOrCreate()" |
142 | 146 | ] |
143 | 147 | }, |
144 | 148 | { |
|
0 commit comments