===== lab08 - Wed Nov  4 01:04:14 PM UTC 2026 =====
$ cd /home/onyxia/work/ece-2026-bigdata
$ echo user-m-anane-ece
user-m-anane-ece
$ aws s3 --profile default ls s3://user-m-anane-ece/warehouse/ --recursive
2026-11-04 13:03:26          0 warehouse/silver/
2026-11-04 13:03:54       1943 warehouse/silver/orders/data/00000-127-d0a7b3e1-c214-4383-b533-ffd3b69869f1-0-00001.parquet
2026-11-04 13:03:32      84055 warehouse/silver/orders/data/00000-43-19316fc3-6a81-4444-a77e-463cc4a7efa3-0-00001.parquet
2026-11-04 13:03:54       7301 warehouse/silver/orders/metadata/661ceae2-4261-4929-8912-3aad37fc6201-m0.avro
2026-11-04 13:03:32       7347 warehouse/silver/orders/metadata/d49e13b5-4371-4dd1-9500-5a87c11804dc-m0.avro
2026-11-04 13:03:54       4525 warehouse/silver/orders/metadata/snap-5530743030737104451-1-661ceae2-4261-4929-8912-3aad37fc6201.avro
2026-11-04 13:03:32       4456 warehouse/silver/orders/metadata/snap-5971824705520254429-1-d49e13b5-4371-4dd1-9500-5a87c11804dc.avro
2026-11-04 13:03:33       1931 warehouse/silver/orders/metadata/v1.metadata.json
2026-11-04 13:03:55       2997 warehouse/silver/orders/metadata/v2.metadata.json
2026-11-04 13:03:55          1 warehouse/silver/orders/metadata/version-hint.text
2026-11-04 13:03:29       7438 warehouse/silver/users/data/00000-39-fae851cc-f83e-4b4b-821a-7462c66fde3f-0-00001.parquet
2026-11-04 13:03:30       7737 warehouse/silver/users/metadata/10e4364a-08cb-40d7-ae28-731b213ecfec-m0.avro
2026-11-04 13:03:30       4453 warehouse/silver/users/metadata/snap-2234938568710704734-1-10e4364a-08cb-40d7-ae28-731b213ecfec.avro
2026-11-04 13:03:30       2200 warehouse/silver/users/metadata/v1.metadata.json
2026-11-04 13:03:31          1 warehouse/silver/users/metadata/version-hint.text
$ uv run pyspark --version
WARNING: Using incubator modules: jdk.incubator.vector
Welcome to
      ____              __
     / __/__  ___ _____/ /__
    _\ \/ _ \/ _ `/ __/  '_/
   /___/ .__/\_,_/_/ /_/\_\   version 4.1.1
      /_/
                        
Using Scala version 2.13.17, OpenJDK 64-Bit Server VM, 21.0.12
Branch HEAD
Compiled by user runner on 2026-01-10T20:56:35Z
Revision c0690c763bafabd08e7079d1137fa0a769a05bae
Url https://github.com/apache/spark.git
Type --help for more information.
$ spark-submit --master 'local[*]' --driver-memory 1g --packages org.apache.iceberg:iceberg-spark-runtime-4.1_2.13:1.11.0 --conf spark.ui.showConsoleProgress=false /home/onyxia/work/labkit/cells.py /home/onyxia/work/labkit/lab08.py
$ grep --line-buffered -vE '^(Ivy|:: |\s+confs|\s+found|\s+org\.|\s+---|\s+\||downloading|\[SUCCESSFUL|resolving|loading settings|Picked up|WARNING: )'
The jars for the packages stored in: /home/onyxia/.ivy2.5.2/jars
org.apache.iceberg#iceberg-spark-runtime-4.1_2.13 added as a dependency
	:: modules in use:
	0 artifacts copied, 1 already retrieved (0kB/33ms)

>>> [cellule 0]
... import os, subprocess
... from pyspark.sql import SparkSession
... from pyspark.sql import functions as F
... def sh(cmd):
...     r = subprocess.run(cmd, shell=True, capture_output=True, text=True)
...     print("$ " + cmd + "\n" + r.stdout + r.stderr)
... 
... os.environ["LAB_BUCKET_NAME"] = os.environ["KUBERNETES_NAMESPACE"]
... BUCKET = os.environ["LAB_BUCKET_NAME"]
... ICEBERG_PACKAGE = "org.apache.iceberg:iceberg-spark-runtime-4.1_2.13:1.11.0"
... spark = (
...     SparkSession.builder
...     .appName("lab-silver-pyspark")
...     .master("local[*]")
...     .config("spark.hadoop.fs.s3a.endpoint", f"https://{os.environ['AWS_S3_ENDPOINT']}")
...     .config("spark.hadoop.fs.s3a.access.key", os.environ["AWS_ACCESS_KEY_ID"])
...     .config("spark.hadoop.fs.s3a.secret.key", os.environ["AWS_SECRET_ACCESS_KEY"])
...     .config("spark.hadoop.fs.s3a.session.token", os.environ.get("AWS_SESSION_TOKEN", ""))
...     .config("spark.hadoop.fs.s3a.aws.credentials.provider", "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider")
...     .config("spark.hadoop.fs.s3a.path.style.access", "true")
...     .config("spark.hadoop.fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem")
...     .config("spark.jars.packages", ICEBERG_PACKAGE)
...     .config("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions")
...     .config("spark.sql.catalog.lab", "org.apache.iceberg.spark.SparkCatalog")
...     .config("spark.sql.catalog.lab.type", "hadoop")
...     .config("spark.sql.catalog.lab.warehouse", f"s3a://{BUCKET}/warehouse")
...     .getOrCreate()
... )
... spark.sparkContext.setLogLevel("ERROR")
... spark.sql("SHOW NAMESPACES IN lab").show()
+---------+
|namespace|
+---------+
|   silver|
+---------+


>>> [cellule 1]
... # La table lab.silver.orders existe deja (lab 07) : pas besoin de la recreer
... spark.sql("SHOW TABLES IN lab.silver").show()
... spark.sql("""
...     SELECT count(*) AS orders,
...            sum(quantity) AS quantity
...     FROM lab.silver.orders
... """).show()
+---------+---------+-----------+
|namespace|tableName|isTemporary|
+---------+---------+-----------+
|   silver|    users|      false|
|   silver|   orders|      false|
+---------+---------+-----------+

+------+--------+
|orders|quantity|
+------+--------+
|  2830|    8617|
+------+--------+


>>> [cellule 2]
... sh(f"aws s3 --profile default ls s3://{BUCKET}/warehouse/silver/orders/ --recursive")
$ aws s3 --profile default ls s3://user-m-anane-ece/warehouse/silver/orders/ --recursive
2026-11-04 13:03:54       1943 warehouse/silver/orders/data/00000-127-d0a7b3e1-c214-4383-b533-ffd3b69869f1-0-00001.parquet
2026-11-04 13:03:32      84055 warehouse/silver/orders/data/00000-43-19316fc3-6a81-4444-a77e-463cc4a7efa3-0-00001.parquet
2026-11-04 13:03:54       7301 warehouse/silver/orders/metadata/661ceae2-4261-4929-8912-3aad37fc6201-m0.avro
2026-11-04 13:03:32       7347 warehouse/silver/orders/metadata/d49e13b5-4371-4dd1-9500-5a87c11804dc-m0.avro
2026-11-04 13:03:54       4525 warehouse/silver/orders/metadata/snap-5530743030737104451-1-661ceae2-4261-4929-8912-3aad37fc6201.avro
2026-11-04 13:03:32       4456 warehouse/silver/orders/metadata/snap-5971824705520254429-1-d49e13b5-4371-4dd1-9500-5a87c11804dc.avro
2026-11-04 13:03:33       1931 warehouse/silver/orders/metadata/v1.metadata.json
2026-11-04 13:03:55       2997 warehouse/silver/orders/metadata/v2.metadata.json
2026-11-04 13:03:55          1 warehouse/silver/orders/metadata/version-hint.text


>>> [cellule 3]
... spark.sql("SELECT * FROM lab.silver.orders.snapshots").show(truncate=False)
... spark.sql("""
...     SELECT file_path, file_format, record_count, file_size_in_bytes
...     FROM lab.silver.orders.files
... """).show(truncate=False)
+-----------------------+-------------------+-------------------+---------+----------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|committed_at           |snapshot_id        |parent_id          |operation|manifest_list                                                                                                               |summary                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
+-----------------------+-------------------+-------------------+---------+----------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|2026-11-04 13:03:32.922|5971824705520254429|NULL               |overwrite|s3a://user-m-anane-ece/warehouse/silver/orders/metadata/snap-5971824705520254429-1-d49e13b5-4371-4dd1-9500-5a87c11804dc.avro|{spark.app.id -> local-1793797372908, manifests-created -> 1, manifests-kept -> 0, manifests-replaced -> 0, added-data-files -> 1, added-records -> 2829, added-files-size -> 84055, changed-partition-count -> 1, total-records -> 2829, total-files-size -> 84055, total-data-files -> 1, total-delete-files -> 0, total-position-deletes -> 0, total-equality-deletes -> 0, engine-version -> 4.1.1, app-id -> local-1793797372908, engine-name -> spark, iceberg-version -> Apache Iceberg 1.11.0 (commit 6976e020b894f6a6777704df2b8c4458cb291ae9), app-name -> lab-silver-pyspark}|
|2026-11-04 13:03:54.933|5530743030737104451|5971824705520254429|append   |s3a://user-m-anane-ece/warehouse/silver/orders/metadata/snap-5530743030737104451-1-661ceae2-4261-4929-8912-3aad37fc6201.avro|{spark.app.id -> local-1793797372908, manifests-created -> 1, manifests-kept -> 1, manifests-replaced -> 0, added-data-files -> 1, added-records -> 1, added-files-size -> 1943, changed-partition-count -> 1, total-records -> 2830, total-files-size -> 85998, total-data-files -> 2, total-delete-files -> 0, total-position-deletes -> 0, total-equality-deletes -> 0, engine-version -> 4.1.1, app-id -> local-1793797372908, engine-name -> spark, iceberg-version -> Apache Iceberg 1.11.0 (commit 6976e020b894f6a6777704df2b8c4458cb291ae9), app-name -> lab-silver-pyspark}    |
+-----------------------+-------------------+-------------------+---------+----------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+

+------------------------------------------------------------------------------------------------------------------+-----------+------------+------------------+
|file_path                                                                                                         |file_format|record_count|file_size_in_bytes|
+------------------------------------------------------------------------------------------------------------------+-----------+------------+------------------+
|s3a://user-m-anane-ece/warehouse/silver/orders/data/00000-127-d0a7b3e1-c214-4383-b533-ffd3b69869f1-0-00001.parquet|PARQUET    |1           |1943              |
|s3a://user-m-anane-ece/warehouse/silver/orders/data/00000-43-19316fc3-6a81-4444-a77e-463cc4a7efa3-0-00001.parquet |PARQUET    |2829        |84055             |
+------------------------------------------------------------------------------------------------------------------+-----------+------------+------------------+


>>> [cellule 4]
... stg_orders = spark.table("lab.silver.orders")
... stg_orders.show(5)
... new_orders = stg_orders.limit(3).withColumn("order_id", F.expr("uuid()"))
... new_orders.writeTo("lab.silver.orders").append()
... spark.sql("""
...     SELECT snapshot_id, operation, committed_at
...     FROM lab.silver.orders.snapshots
...     ORDER BY committed_at
... """).show(truncate=False)
... spark.sql("SELECT count(*) FROM lab.silver.orders").show()
... spark.sql("SELECT count(*) AS data_files FROM lab.silver.orders.files").show()
+--------------------+--------------------+--------------------+--------+---------+
|            order_id|             user_id|          ordered_at|quantity|  product|
+--------------------+--------------------+--------------------+--------+---------+
|223b6b52-3dbd-4ec...|f53d4330-cdda-44b...|2020-01-28 12:31:...|       2|   cookie|
|00060d5a-02b3-494...|f53d4330-cdda-44b...|2020-01-28 12:31:...|       2|   cookie|
|0011be30-e268-492...|5f733a3e-5d28-4aa...|2020-04-18 22:51:...|       5|    bread|
|002394e2-2ad9-468...|90164161-cfa7-41c...|2020-02-24 01:03:...|       4|   cookie|
|0041f5d4-9549-463...|aab612c9-415d-474...|2020-03-17 18:29:...|       2|croissant|
+--------------------+--------------------+--------------------+--------+---------+
only showing top 5 rows
+-------------------+---------+-----------------------+
|snapshot_id        |operation|committed_at           |
+-------------------+---------+-----------------------+
|5971824705520254429|overwrite|2026-11-04 13:03:32.922|
|5530743030737104451|append   |2026-11-04 13:03:54.933|
|1614423503086333191|append   |2026-11-04 13:04:56.68 |
+-------------------+---------+-----------------------+

+--------+
|count(1)|
+--------+
|    2833|
+--------+

+----------+
|data_files|
+----------+
|         3|
+----------+


>>> [cellule 5]
... snapshots = spark.sql("""
...     SELECT snapshot_id
...     FROM lab.silver.orders.snapshots
...     ORDER BY committed_at
... """).collect()
... snapshots
[Row(snapshot_id=5971824705520254429), Row(snapshot_id=5530743030737104451), Row(snapshot_id=1614423503086333191)]

>>> [cellule 6]
... first_snapshot = snapshots[0]["snapshot_id"]
... spark.sql(f"""
...     SELECT count(*)
...     FROM lab.silver.orders
...     VERSION AS OF {first_snapshot}
... """).show()
... spark.sql("SELECT count(*) FROM lab.silver.orders").show()
+--------+
|count(1)|
+--------+
|    2829|
+--------+

+--------+
|count(1)|
+--------+
|    2833|
+--------+


>>> [cellule 7]
... spark.sql("ALTER TABLE lab.silver.orders ADD COLUMN source STRING")
... spark.sql("DESCRIBE lab.silver.orders").show()
... spark.sql("""
...     INSERT INTO lab.silver.orders
...     SELECT order_id, user_id, ordered_at, quantity, product, 'spark'
...     FROM lab.silver.orders
...     LIMIT 1
... """)
... spark.sql("SELECT source, count(*) FROM lab.silver.orders GROUP BY source").show()
... sh(f"aws s3 --profile default ls s3://{BUCKET}/warehouse/silver/orders/ --recursive")
+----------+---------+-------+
|  col_name|data_type|comment|
+----------+---------+-------+
|  order_id|   string|   NULL|
|   user_id|   string|   NULL|
|ordered_at|timestamp|   NULL|
|  quantity|      int|   NULL|
|   product|   string|   NULL|
|    source|   string|   NULL|
+----------+---------+-------+

+------+--------+
|source|count(1)|
+------+--------+
|  NULL|    2833|
| spark|       1|
+------+--------+

$ aws s3 --profile default ls s3://user-m-anane-ece/warehouse/silver/orders/ --recursive
2026-11-04 13:03:54       1943 warehouse/silver/orders/data/00000-127-d0a7b3e1-c214-4383-b533-ffd3b69869f1-0-00001.parquet
2026-11-04 13:05:03       2201 warehouse/silver/orders/data/00000-20-cfe454fa-228e-4f3a-8e72-08465e913dbf-0-00001.parquet
2026-11-04 13:03:32      84055 warehouse/silver/orders/data/00000-43-19316fc3-6a81-4444-a77e-463cc4a7efa3-0-00001.parquet
2026-11-04 13:04:56       2048 warehouse/silver/orders/data/00000-6-4be77fc1-00ed-4fcf-b293-0e2f99efd52e-0-00001.parquet
2026-11-04 13:03:54       7301 warehouse/silver/orders/metadata/661ceae2-4261-4929-8912-3aad37fc6201-m0.avro
2026-11-04 13:05:03       7365 warehouse/silver/orders/metadata/a161b5b0-25ef-4dfe-89be-8850c3dec1ab-m0.avro
2026-11-04 13:04:56       7333 warehouse/silver/orders/metadata/b2bc5c08-f0dc-456d-90a7-8415334d14d3-m0.avro
2026-11-04 13:03:32       7347 warehouse/silver/orders/metadata/d49e13b5-4371-4dd1-9500-5a87c11804dc-m0.avro
2026-11-04 13:04:56       4581 warehouse/silver/orders/metadata/snap-1614423503086333191-1-b2bc5c08-f0dc-456d-90a7-8415334d14d3.avro
2026-11-04 13:03:54       4525 warehouse/silver/orders/metadata/snap-5530743030737104451-1-661ceae2-4261-4929-8912-3aad37fc6201.avro
2026-11-04 13:05:03       4623 warehouse/silver/orders/metadata/snap-5785441293415314053-1-a161b5b0-25ef-4dfe-89be-8850c3dec1ab.avro
2026-11-04 13:03:32       4456 warehouse/silver/orders/metadata/snap-5971824705520254429-1-d49e13b5-4371-4dd1-9500-5a87c11804dc.avro
2026-11-04 13:03:33       1931 warehouse/silver/orders/metadata/v1.metadata.json
2026-11-04 13:03:55       2997 warehouse/silver/orders/metadata/v2.metadata.json
2026-11-04 13:04:56       4064 warehouse/silver/orders/metadata/v3.metadata.json
2026-11-04 13:05:02       4589 warehouse/silver/orders/metadata/v4.metadata.json
2026-11-04 13:05:04       5656 warehouse/silver/orders/metadata/v5.metadata.json
2026-11-04 13:05:04          1 warehouse/silver/orders/metadata/version-hint.text


>>> [cellule 8]
... # Comparaison Iceberg / Parquet
... sh(f"aws s3 --profile default ls s3://{BUCKET}/silver/spark/orders.parquet/ --recursive")
$ aws s3 --profile default ls s3://user-m-anane-ece/silver/spark/orders.parquet/ --recursive
2026-11-04 13:03:58          0 silver/spark/orders.parquet/
2026-11-04 13:03:59      11803 silver/spark/orders.parquet/_SUCCESS
2026-11-04 13:03:59       1978 silver/spark/orders.parquet/part-00000-c44adbc4-b6b9-4c78-899e-b9bc80eeff37-c000.snappy.parquet


>>> [cellule 9]
... # Exercices 1 a 4 sur lab.silver.users
... spark.sql("SHOW TABLES IN lab.silver").show()
... users = spark.table("lab.silver.users")
... print("users:", users.count())
... batch = users.limit(2).withColumn("user_id", F.expr("uuid()")).withColumn("username", F.concat(F.col("username"), F.lit("_new")))
... batch.writeTo("lab.silver.users").append()
... snaps = spark.sql("SELECT snapshot_id, operation, committed_at FROM lab.silver.users.snapshots ORDER BY committed_at")
... snaps.show(truncate=False)
... s = snaps.collect()
... spark.sql(f"SELECT count(*) AS before_append FROM lab.silver.users VERSION AS OF {s[0]['snapshot_id']}").show()
... spark.sql(f"SELECT count(*) AS after_append FROM lab.silver.users VERSION AS OF {s[-1]['snapshot_id']}").show()
... spark.sql("ALTER TABLE lab.silver.users ADD COLUMN country STRING")
... spark.table("lab.silver.users").limit(1).withColumn("user_id", F.expr("uuid()")).withColumn("country", F.lit("US")).writeTo("lab.silver.users").append()
... spark.sql("SELECT country, count(*) FROM lab.silver.users GROUP BY country").show()
+---------+---------+-----------+
|namespace|tableName|isTemporary|
+---------+---------+-----------+
|   silver|    users|      false|
|   silver|   orders|      false|
+---------+---------+-----------+

users: 50
+-------------------+---------+-----------------------+
|snapshot_id        |operation|committed_at           |
+-------------------+---------+-----------------------+
|2234938568710704734|overwrite|2026-11-04 13:03:30.259|
|4831401298753088140|append   |2026-11-04 13:05:12.564|
+-------------------+---------+-----------------------+

+-------------+
|before_append|
+-------------+
|           50|
+-------------+

+------------+
|after_append|
+------------+
|          52|
+------------+

+-------+--------+
|country|count(1)|
+-------+--------+
|   NULL|      52|
|     US|       1|
+-------+--------+


>>> [cellule 10]
... # Exercice 5 : layout S3 des deux tables
... sh(f"aws s3 --profile default ls s3://{BUCKET}/warehouse/silver/ --recursive")
$ aws s3 --profile default ls s3://user-m-anane-ece/warehouse/silver/ --recursive
2026-11-04 13:03:26          0 warehouse/silver/
2026-11-04 13:03:54       1943 warehouse/silver/orders/data/00000-127-d0a7b3e1-c214-4383-b533-ffd3b69869f1-0-00001.parquet
2026-11-04 13:05:03       2201 warehouse/silver/orders/data/00000-20-cfe454fa-228e-4f3a-8e72-08465e913dbf-0-00001.parquet
2026-11-04 13:03:32      84055 warehouse/silver/orders/data/00000-43-19316fc3-6a81-4444-a77e-463cc4a7efa3-0-00001.parquet
2026-11-04 13:04:56       2048 warehouse/silver/orders/data/00000-6-4be77fc1-00ed-4fcf-b293-0e2f99efd52e-0-00001.parquet
2026-11-04 13:03:54       7301 warehouse/silver/orders/metadata/661ceae2-4261-4929-8912-3aad37fc6201-m0.avro
2026-11-04 13:05:03       7365 warehouse/silver/orders/metadata/a161b5b0-25ef-4dfe-89be-8850c3dec1ab-m0.avro
2026-11-04 13:04:56       7333 warehouse/silver/orders/metadata/b2bc5c08-f0dc-456d-90a7-8415334d14d3-m0.avro
2026-11-04 13:03:32       7347 warehouse/silver/orders/metadata/d49e13b5-4371-4dd1-9500-5a87c11804dc-m0.avro
2026-11-04 13:04:56       4581 warehouse/silver/orders/metadata/snap-1614423503086333191-1-b2bc5c08-f0dc-456d-90a7-8415334d14d3.avro
2026-11-04 13:03:54       4525 warehouse/silver/orders/metadata/snap-5530743030737104451-1-661ceae2-4261-4929-8912-3aad37fc6201.avro
2026-11-04 13:05:03       4623 warehouse/silver/orders/metadata/snap-5785441293415314053-1-a161b5b0-25ef-4dfe-89be-8850c3dec1ab.avro
2026-11-04 13:03:32       4456 warehouse/silver/orders/metadata/snap-5971824705520254429-1-d49e13b5-4371-4dd1-9500-5a87c11804dc.avro
2026-11-04 13:03:33       1931 warehouse/silver/orders/metadata/v1.metadata.json
2026-11-04 13:03:55       2997 warehouse/silver/orders/metadata/v2.metadata.json
2026-11-04 13:04:56       4064 warehouse/silver/orders/metadata/v3.metadata.json
2026-11-04 13:05:02       4589 warehouse/silver/orders/metadata/v4.metadata.json
2026-11-04 13:05:04       5656 warehouse/silver/orders/metadata/v5.metadata.json
2026-11-04 13:05:04          1 warehouse/silver/orders/metadata/version-hint.text
2026-11-04 13:05:12       3245 warehouse/silver/users/data/00000-26-f1da5d6f-ba63-4c2c-b314-2e31eb4889fa-0-00001.parquet
2026-11-04 13:05:16       3572 warehouse/silver/users/data/00000-36-3457baf6-4ef2-4182-b78e-72346ae3b384-0-00001.parquet
2026-11-04 13:03:29       7438 warehouse/silver/users/data/00000-39-fae851cc-f83e-4b4b-821a-7462c66fde3f-0-00001.parquet
2026-11-04 13:05:16       7730 warehouse/silver/users/metadata/0983496e-b297-4437-bf85-8e29a1055b35-m0.avro
2026-11-04 13:03:30       7737 warehouse/silver/users/metadata/10e4364a-08cb-40d7-ae28-731b213ecfec-m0.avro
2026-11-04 13:05:12       7730 warehouse/silver/users/metadata/8df624d2-bde7-4655-bb29-a68b3157166e-m0.avro
2026-11-04 13:03:30       4453 warehouse/silver/users/metadata/snap-2234938568710704734-1-10e4364a-08cb-40d7-ae28-731b213ecfec.avro
2026-11-04 13:05:12       4528 warehouse/silver/users/metadata/snap-4831401298753088140-1-8df624d2-bde7-4655-bb29-a68b3157166e.avro
2026-11-04 13:05:16       4575 warehouse/silver/users/metadata/snap-6558679593786463913-1-0983496e-b297-4437-bf85-8e29a1055b35.avro
2026-11-04 13:03:30       2200 warehouse/silver/users/metadata/v1.metadata.json
2026-11-04 13:05:12       3262 warehouse/silver/users/metadata/v2.metadata.json
2026-11-04 13:05:15       4064 warehouse/silver/users/metadata/v3.metadata.json
2026-11-04 13:05:16       5127 warehouse/silver/users/metadata/v4.metadata.json
2026-11-04 13:05:16          1 warehouse/silver/users/metadata/version-hint.text


>>> [cellule 11]
... # Retour au schema initial de users (le lab 09 ajoute lui-meme une colonne country)
... spark.sql("ALTER TABLE lab.silver.users DROP COLUMN country")
... spark.sql("DESCRIBE lab.silver.users").show()
+---------+---------+-------+
| col_name|data_type|comment|
+---------+---------+-------+
|  user_id|   string|   NULL|
| username|   string|   NULL|
|     name|   string|   NULL|
|      sex|   string|   NULL|
|    email|   string|   NULL|
|birthdate|     date|   NULL|
|   street|   string|   NULL|
|     city|   string|   NULL|
|    state|   string|   NULL|
| zip_code|   string|   NULL|
+---------+---------+-------+


>>> [cellule 12]
... spark.stop()
$ aws s3 --profile default ls s3://user-m-anane-ece/warehouse/silver/ --recursive
$ awk '{print $4}'
$ sed 's#/[^/]*$##'
$ sort
$ uniq -c
      1 warehouse/silver
      4 warehouse/silver/orders/data
     14 warehouse/silver/orders/metadata
      3 warehouse/silver/users/data
     12 warehouse/silver/users/metadata
$ true
EXIT_CODE=0