diff --git a/notebooks/bigquery.ipynb b/notebooks/bigquery.ipynb index 750f167..45b5400 100644 --- a/notebooks/bigquery.ipynb +++ b/notebooks/bigquery.ipynb @@ -15,21 +15,21 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 1/1 [00:00<00:00, 5440.08it/s]\n" + "100%|██████████| 1/1 [00:00<00:00, 10106.76it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "['data transport version ', '2.0.0']\n" + "['data transport version ', '2.0.4']\n" ] } ], @@ -45,7 +45,7 @@ "PRIVATE_KEY = os.environ['BQ_KEY'] #-- location of the service key\n", "DATASET = 'demo'\n", "_data = pd.DataFrame({\"name\":['James Bond','Steve Rogers','Steve Nyemba'],'age':[55,150,44]})\n", - "bqw = transport.factory.instance(provider=providers.BIGQUERY,dataset=DATASET,table='friends',context='write',private_key=PRIVATE_KEY)\n", + "bqw = transport.get.writer(provider=providers.BIGQUERY,dataset=DATASET,table='friends',private_key=PRIVATE_KEY)\n", "bqw.write(_data,if_exists='replace') #-- default is append\n", "print (['data transport version ', transport.__version__])\n" ] @@ -63,7 +63,8 @@ "\n", "**NOTE**\n", "\n", - "It is possible to use **transport.factory.instance** or **transport.instance** they are the same. It allows the maintainers to know that we used a factory design pattern." + "By design **read** object are separated from **write** objects in order to avoid accidental writes to the database.\n", + "Read objects are created with **transport.get.reader** whereas write objects are created with **transport.get.writer**" ] }, { @@ -93,7 +94,7 @@ "from transport import providers\n", "import os\n", "PRIVATE_KEY=os.environ['BQ_KEY']\n", - "pgr = transport.instance(provider=providers.BIGQUERY,dataset='demo',table='friends',private_key=PRIVATE_KEY)\n", + "pgr = transport.get.reader(provider=providers.BIGQUERY,dataset='demo',table='friends',private_key=PRIVATE_KEY)\n", "_df = pgr.read()\n", "_query = 'SELECT COUNT(*) _counts, AVG(age) from demo.friends'\n", "_sdf = pgr.read(sql=_query)\n", @@ -106,35 +107,13 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The cell bellow show the content of an auth_file, in this case if the dataset/table in question is not to be shared then you can use auth_file with information associated with the parameters.\n", + "An **auth-file** is a file that contains database parameters used to access the database. \n", + "For code in shared environments, we recommend \n", "\n", - "**NOTE**:\n", + "1. Having the **auth-file** stored on disk \n", + "2. and the location of the file is set to an environment variable.\n", "\n", - "The auth_file is intended to be **JSON** formatted" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'dataset': 'demo', 'table': 'friends'}" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "\n", - "{\n", - " \n", - " \"dataset\":\"demo\",\"table\":\"friends\"\n", - "}" + "To generate a template of the **auth-file** open the **file generator wizard** found at visit https://healthcareio.the-phi.com/data-transport" ] }, { diff --git a/notebooks/mongodb.ipynb b/notebooks/mongodb.ipynb index 0554669..fb1532c 100644 --- a/notebooks/mongodb.ipynb +++ b/notebooks/mongodb.ipynb @@ -11,14 +11,14 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "2.0.0\n" + "2.0.4\n" ] } ], @@ -30,7 +30,7 @@ "from transport import providers\n", "import pandas as pd\n", "_data = pd.DataFrame({\"name\":['James Bond','Steve Rogers','Steve Nyemba'],'age':[55,150,44]})\n", - "mgw = transport.factory.instance(provider=providers.MONGODB,db='demo',collection='friends',context='write')\n", + "mgw = transport.get.writer(provider=providers.MONGODB,db='demo',collection='friends')\n", "mgw.write(_data)\n", "print (transport.__version__)" ] @@ -48,12 +48,13 @@ "\n", "**NOTE**\n", "\n", - "It is possible to use **transport.factory.instance** or **transport.instance** they are the same. It allows the maintainers to know that we used a factory design pattern." + "By design **read** object are separated from **write** objects in order to avoid accidental writes to the database.\n", + "Read objects are created with **transport.get.reader** whereas write objects are created with **transport.get.writer**" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -73,7 +74,7 @@ "\n", "import transport\n", "from transport import providers\n", - "mgr = transport.instance(provider=providers.MONGODB,db='foo',collection='friends')\n", + "mgr = transport.get.reader(provider=providers.MONGODB,db='foo',collection='friends')\n", "_df = mgr.read()\n", "PIPELINE = [{\"$group\":{\"_id\":0,\"_counts\":{\"$sum\":1}, \"_mean\":{\"$avg\":\"$age\"}}}]\n", "_sdf = mgr.read(aggregate='friends',pipeline=PIPELINE)\n", @@ -86,41 +87,13 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The cell bellow show the content of an auth_file, in this case if the dataset/table in question is not to be shared then you can use auth_file with information associated with the parameters.\n", + "An **auth-file** is a file that contains database parameters used to access the database. \n", + "For code in shared environments, we recommend \n", "\n", - "**NOTE**:\n", + "1. Having the **auth-file** stored on disk \n", + "2. and the location of the file is set to an environment variable.\n", "\n", - "The auth_file is intended to be **JSON** formatted" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'host': 'klingon.io',\n", - " 'port': 27017,\n", - " 'username': 'me',\n", - " 'password': 'foobar',\n", - " 'db': 'foo',\n", - " 'collection': 'friends',\n", - " 'authSource': '',\n", - " 'mechamism': ''}" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "{\n", - " \"host\":\"klingon.io\",\"port\":27017,\"username\":\"me\",\"password\":\"foobar\",\"db\":\"foo\",\"collection\":\"friends\",\n", - " \"authSource\":\"\",\"mechamism\":\"\"\n", - "}" + "To generate a template of the **auth-file** open the **file generator wizard** found at visit https://healthcareio.the-phi.com/data-transport" ] }, { diff --git a/notebooks/mssqlserver.ipynb b/notebooks/mssqlserver.ipynb index f2bee85..51827b3 100644 --- a/notebooks/mssqlserver.ipynb +++ b/notebooks/mssqlserver.ipynb @@ -17,17 +17,9 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['data transport version ', '2.0.0']\n" - ] - } - ], + "outputs": [], "source": [ "#\n", "# Writing to Google Bigquery database\n", @@ -41,7 +33,7 @@ "MSSQL_AUTH_FILE= os.sep.join([AUTH_FOLDER,'mssql.json'])\n", "\n", "_data = pd.DataFrame({\"name\":['James Bond','Steve Rogers','Steve Nyemba'],'age':[55,150,44]})\n", - "msw = transport.factory.instance(provider=providers.MSSQL,table='friends',context='write',auth_file=MSSQL_AUTH_FILE)\n", + "msw = transport.get.writer(provider=providers.MSSQL,table='friends',auth_file=MSSQL_AUTH_FILE)\n", "msw.write(_data,if_exists='replace') #-- default is append\n", "print (['data transport version ', transport.__version__])\n" ] @@ -59,30 +51,15 @@ "\n", "**NOTE**\n", "\n", - "It is possible to use **transport.factory.instance** or **transport.instance** they are the same. It allows the maintainers to know that we used a factory design pattern." + "By design **read** object are separated from **write** objects in order to avoid accidental writes to the database.\n", + "Read objects are created with **transport.get.reader** whereas write objects are created with **transport.get.writer**" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " name age\n", - "0 James Bond 55\n", - "1 Steve Rogers 150\n", - "2 Steve Nyemba 44\n", - "\n", - "--------- STATISTICS ------------\n", - "\n", - " _counts \n", - "0 3 83\n" - ] - } - ], + "outputs": [], "source": [ "\n", "import transport\n", @@ -91,7 +68,7 @@ "AUTH_FOLDER = os.environ['DT_AUTH_FOLDER'] #-- location of the service key\n", "MSSQL_AUTH_FILE= os.sep.join([AUTH_FOLDER,'mssql.json'])\n", "\n", - "msr = transport.instance(provider=providers.MSSQL,table='friends',auth_file=MSSQL_AUTH_FILE)\n", + "msr = transport.get.reader(provider=providers.MSSQL,table='friends',auth_file=MSSQL_AUTH_FILE)\n", "_df = msr.read()\n", "_query = 'SELECT COUNT(*) _counts, AVG(age) from friends'\n", "_sdf = msr.read(sql=_query)\n", @@ -104,25 +81,31 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The cell bellow show the content of an auth_file, in this case if the dataset/table in question is not to be shared then you can use auth_file with information associated with the parameters.\n", + "An **auth-file** is a file that contains database parameters used to access the database. \n", + "For code in shared environments, we recommend \n", "\n", - "**NOTE**:\n", + "1. Having the **auth-file** stored on disk \n", + "2. and the location of the file is set to an environment variable.\n", "\n", - "The auth_file is intended to be **JSON** formatted" + "To generate a template of the **auth-file** open the **file generator wizard** found at visit https://healthcareio.the-phi.com/data-transport" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 1, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'dataset': 'demo', 'table': 'friends'}" + "{'provider': 'sqlserver',\n", + " 'dataset': 'demo',\n", + " 'table': 'friends',\n", + " 'username': '',\n", + " 'password': ''}" ] }, - "execution_count": 3, + "execution_count": 1, "metadata": {}, "output_type": "execute_result" } @@ -130,10 +113,17 @@ "source": [ "\n", "{\n", - " \n", + " \"provider\":\"sqlserver\",\n", " \"dataset\":\"demo\",\"table\":\"friends\",\"username\":\"\",\"password\":\"\"\n", "}" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/notebooks/mysql.ipynb b/notebooks/mysql.ipynb index a54d46d..4b8fb6b 100644 --- a/notebooks/mysql.ipynb +++ b/notebooks/mysql.ipynb @@ -14,14 +14,14 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "2.0.0\n" + "2.0.4\n" ] } ], @@ -33,7 +33,7 @@ "from transport import providers\n", "import pandas as pd\n", "_data = pd.DataFrame({\"name\":['James Bond','Steve Rogers','Steve Nyemba'],'age':[55,150,44]})\n", - "myw = transport.factory.instance(provider=providers.MYSQL,database='demo',table='friends',context='write',auth_file=\"/home/steve/auth-mysql.json\")\n", + "myw = transport.get.writer(provider=providers.MYSQL,database='demo',table='friends',auth_file=\"/home/steve/auth-mysql.json\")\n", "myw.write(_data,if_exists='replace') #-- default is append\n", "print (transport.__version__)" ] @@ -51,12 +51,13 @@ "\n", "**NOTE**\n", "\n", - "It is possible to use **transport.factory.instance** or **transport.instance** they are the same. It allows the maintainers to know that we used a factory design pattern." + "By design **read** object are separated from **write** objects in order to avoid accidental writes to the database.\n", + "Read objects are created with **transport.get.reader** whereas write objects are created with **transport.get.writer**" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -68,8 +69,8 @@ "1 Steve Rogers 150\n", "2 Steve Nyemba 44\n", "--------- STATISTICS ------------\n", - " _counts avg\n", - "0 3 83.0\n" + " _counts AVG(age)\n", + "0 3 83.0\n" ] } ], @@ -77,7 +78,7 @@ "\n", "import transport\n", "from transport import providers\n", - "myr = transport.instance(provider=providers.POSTGRESQL,database='demo',table='friends',auth_file='/home/steve/auth-mysql.json')\n", + "myr = transport.get.reader(provider=providers.MYSQL,database='demo',table='friends',auth_file='/home/steve/auth-mysql.json')\n", "_df = myr.read()\n", "_query = 'SELECT COUNT(*) _counts, AVG(age) from friends'\n", "_sdf = myr.read(sql=_query)\n", @@ -90,16 +91,18 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The cell bellow show the content of an auth_file, in this case if the dataset/table in question is not to be shared then you can use auth_file with information associated with the parameters.\n", + "An **auth-file** is a file that contains database parameters used to access the database. \n", + "For code in shared environments, we recommend \n", "\n", - "**NOTE**:\n", + "1. Having the **auth-file** stored on disk \n", + "2. and the location of the file is set to an environment variable.\n", "\n", - "The auth_file is intended to be **JSON** formatted" + "To generate a template of the **auth-file** open the **file generator wizard** found at visit https://healthcareio.the-phi.com/data-transport" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -109,21 +112,29 @@ " 'port': 3306,\n", " 'username': 'me',\n", " 'password': 'foobar',\n", + " 'provider': 'mysql',\n", " 'database': 'demo',\n", " 'table': 'friends'}" ] }, - "execution_count": 1, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "{\n", - " \"host\":\"klingon.io\",\"port\":3306,\"username\":\"me\",\"password\":\"foobar\",\n", + " \"host\":\"klingon.io\",\"port\":3306,\"username\":\"me\",\"password\":\"foobar\", \"provider\":\"mysql\",\n", " \"database\":\"demo\",\"table\":\"friends\"\n", "}" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/notebooks/postgresql.ipynb b/notebooks/postgresql.ipynb index 5046f4d..85f4322 100644 --- a/notebooks/postgresql.ipynb +++ b/notebooks/postgresql.ipynb @@ -14,14 +14,14 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "2.0.0\n" + "2.0.4\n" ] } ], @@ -33,7 +33,7 @@ "from transport import providers\n", "import pandas as pd\n", "_data = pd.DataFrame({\"name\":['James Bond','Steve Rogers','Steve Nyemba'],'age':[55,150,44]})\n", - "pgw = transport.factory.instance(provider=providers.POSTGRESQL,database='demo',table='friends',context='write')\n", + "pgw = transport.get.writer(provider=providers.POSTGRESQL,database='demo',table='friends')\n", "pgw.write(_data,if_exists='replace') #-- default is append\n", "print (transport.__version__)" ] @@ -49,14 +49,16 @@ "- Basic read of the designated table (friends) created above\n", "- Execute an aggregate SQL against the table\n", "\n", + "\n", "**NOTE**\n", "\n", - "It is possible to use **transport.factory.instance** or **transport.instance** they are the same. It allows the maintainers to know that we used a factory design pattern." + "By design **read** object are separated from **write** objects in order to avoid accidental writes to the database.\n", + "Read objects are created with **transport.get.reader** whereas write objects are created with **transport.get.writer**" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -77,7 +79,7 @@ "\n", "import transport\n", "from transport import providers\n", - "pgr = transport.instance(provider=providers.POSTGRESQL,database='demo',table='friends')\n", + "pgr = transport.get.reader(provider=providers.POSTGRESQL,database='demo',table='friends')\n", "_df = pgr.read()\n", "_query = 'SELECT COUNT(*) _counts, AVG(age) from friends'\n", "_sdf = pgr.read(sql=_query)\n", @@ -90,16 +92,18 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The cell bellow show the content of an auth_file, in this case if the dataset/table in question is not to be shared then you can use auth_file with information associated with the parameters.\n", + "An **auth-file** is a file that contains database parameters used to access the database. \n", + "For code in shared environments, we recommend \n", "\n", - "**NOTE**:\n", + "1. Having the **auth-file** stored on disk \n", + "2. and the location of the file is set to an environment variable.\n", "\n", - "The auth_file is intended to be **JSON** formatted" + "To generate a template of the **auth-file** open the **file generator wizard** found at visit https://healthcareio.the-phi.com/data-transport" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -109,18 +113,19 @@ " 'port': 5432,\n", " 'username': 'me',\n", " 'password': 'foobar',\n", + " 'provider': 'postgresql',\n", " 'database': 'demo',\n", " 'table': 'friends'}" ] }, - "execution_count": 1, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "{\n", - " \"host\":\"klingon.io\",\"port\":5432,\"username\":\"me\",\"password\":\"foobar\",\n", + " \"host\":\"klingon.io\",\"port\":5432,\"username\":\"me\",\"password\":\"foobar\", \"provider\":\"postgresql\",\n", " \"database\":\"demo\",\"table\":\"friends\"\n", "}" ] diff --git a/notebooks/sqlite.ipynb b/notebooks/sqlite.ipynb index 5c249de..2a836c5 100644 --- a/notebooks/sqlite.ipynb +++ b/notebooks/sqlite.ipynb @@ -18,7 +18,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "2.0.0\n" + "2.0.4\n" ] } ], @@ -30,7 +30,7 @@ "from transport import providers\n", "import pandas as pd\n", "_data = pd.DataFrame({\"name\":['James Bond','Steve Rogers','Steve Nyemba'],'age':[55,150,44]})\n", - "sqw = transport.factory.instance(provider=providers.SQLITE,database='/home/steve/demo.db3',table='friends',context='write')\n", + "sqw = transport.get.writer(provider=providers.SQLITE,database='/home/steve/demo.db3',table='friends')\n", "sqw.write(_data,if_exists='replace') #-- default is append\n", "print (transport.__version__)" ] @@ -46,9 +46,11 @@ "- Basic read of the designated table (friends) created above\n", "- Execute an aggregate SQL against the table\n", "\n", + "\n", "**NOTE**\n", "\n", - "It is possible to use **transport.factory.instance** or **transport.instance** they are the same. It allows the maintainers to know that we used a factory design pattern." + "By design **read** object are separated from **write** objects in order to avoid accidental writes to the database.\n", + "Read objects are created with **transport.get.reader** whereas write objects are created with **transport.get.writer**" ] }, { @@ -74,10 +76,10 @@ "\n", "import transport\n", "from transport import providers\n", - "pgr = transport.instance(provider=providers.SQLITE,database='/home/steve/demo.db3',table='friends')\n", - "_df = pgr.read()\n", + "sqr = transport.get.reader(provider=providers.SQLITE,database='/home/steve/demo.db3',table='friends')\n", + "_df = sqr.read()\n", "_query = 'SELECT COUNT(*) _counts, AVG(age) from friends'\n", - "_sdf = pgr.read(sql=_query)\n", + "_sdf = sqr.read(sql=_query)\n", "print (_df)\n", "print ('--------- STATISTICS ------------')\n", "print (_sdf)" @@ -87,11 +89,13 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The cell bellow show the content of an auth_file, in this case if the dataset/table in question is not to be shared then you can use auth_file with information associated with the parameters.\n", + "An **auth-file** is a file that contains database parameters used to access the database. \n", + "For code in shared environments, we recommend \n", "\n", - "**NOTE**:\n", + "1. Having the **auth-file** stored on disk \n", + "2. and the location of the file is set to an environment variable.\n", "\n", - "The auth_file is intended to be **JSON** formatted. This is an overkill for SQLite ;-)" + "To generate a template of the **auth-file** open the **file generator wizard** found at visit https://healthcareio.the-phi.com/data-transport" ] }, {