Update a dataset's partition spec
curl --request POST \
--url https://api-dev.narrative.io/datasets/{dataset_id}/admin/partition-spec \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"updates": [
{
"type": "add",
"field": "id.value",
"transform": "bucket[128]"
},
{
"type": "remove",
"field": "__NARRATIVE_DEFAULT_PARTITION___bucket"
}
]
}
'import requests
url = "https://api-dev.narrative.io/datasets/{dataset_id}/admin/partition-spec"
payload = { "updates": [
{
"type": "add",
"field": "id.value",
"transform": "bucket[128]"
},
{
"type": "remove",
"field": "__NARRATIVE_DEFAULT_PARTITION___bucket"
}
] }
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
updates: [
{type: 'add', field: 'id.value', transform: 'bucket[128]'},
{type: 'remove', field: '__NARRATIVE_DEFAULT_PARTITION___bucket'}
]
})
};
fetch('https://api-dev.narrative.io/datasets/{dataset_id}/admin/partition-spec', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api-dev.narrative.io/datasets/{dataset_id}/admin/partition-spec",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'updates' => [
[
'type' => 'add',
'field' => 'id.value',
'transform' => 'bucket[128]'
],
[
'type' => 'remove',
'field' => '__NARRATIVE_DEFAULT_PARTITION___bucket'
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api-dev.narrative.io/datasets/{dataset_id}/admin/partition-spec"
payload := strings.NewReader("{\n \"updates\": [\n {\n \"type\": \"add\",\n \"field\": \"id.value\",\n \"transform\": \"bucket[128]\"\n },\n {\n \"type\": \"remove\",\n \"field\": \"__NARRATIVE_DEFAULT_PARTITION___bucket\"\n }\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api-dev.narrative.io/datasets/{dataset_id}/admin/partition-spec")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"updates\": [\n {\n \"type\": \"add\",\n \"field\": \"id.value\",\n \"transform\": \"bucket[128]\"\n },\n {\n \"type\": \"remove\",\n \"field\": \"__NARRATIVE_DEFAULT_PARTITION___bucket\"\n }\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api-dev.narrative.io/datasets/{dataset_id}/admin/partition-spec")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"updates\": [\n {\n \"type\": \"add\",\n \"field\": \"id.value\",\n \"transform\": \"bucket[128]\"\n },\n {\n \"type\": \"remove\",\n \"field\": \"__NARRATIVE_DEFAULT_PARTITION___bucket\"\n }\n ]\n}"
response = http.request(request)
puts response.read_body{
"id": 18923,
"company_id": 382,
"created_at": "2021-06-24T00:54:40.029056Z",
"data_plane": {
"type": "owned",
"id": "c0bed5c2-8af8-4f7a-9787-1ffc3def339b",
"company_id": 382,
"created_at": "2021-06-01T00:00:00Z",
"description": "The company's Snowflake account",
"display_name": "Snowflake (us-west-2)",
"external_id": "snowflake-prod",
"platform": {
"type": "platform_snowflake",
"account_locator": "ab12345",
"account_name": "acme",
"organization_name": "acmeorg",
"region": {
"type": "region_aws",
"id": "us-west-2"
}
},
"status": "active",
"tags": [],
"updated_at": "2021-06-01T00:00:00Z",
"collaborators": {
"participants": {
"type": "all"
},
"manage_compute_pools": {
"type": "all"
}
},
"compute_pools": []
},
"is_narrative_managed": true,
"retention_policy": {
"policies": [
{
"type": {
"kind": "snapshot_ttl",
"snapshot_retention_policy": {
"type": "retain_everything"
}
},
"enabled": true
}
]
},
"description": "Mobile identifiers tied to birth information provided as part of app registration.",
"name": "declared_age",
"display_name": "Declared Age",
"schema": {
"file_config": {
"type": "flat",
"delimiter": ",",
"escape": "\"",
"header": true,
"quote": "\""
},
"type": "object",
"properties": {
"idValue": {
"display_name": "id value",
"order": 0,
"approximate_cardinality": 100000,
"description": "ID values",
"type": "string"
},
"idType": {
"display_name": "id type",
"order": 1,
"approximate_cardinality": 1,
"type": "string",
"enum": [
"md5_email",
"sha256_email"
]
},
"eventTimestamp": {
"display_name": "event timestamp",
"order": 2,
"description": "When the observation was collected",
"type": "timestamptz",
"validations": [
"eventTimestamp < current_timestamp()"
]
},
"countryCode": {
"display_name": "country code",
"order": 3,
"approximate_cardinality": 6,
"description": "The country of residence of the individual",
"type": "string"
},
"age": {
"display_name": "age",
"order": 4,
"approximate_cardinality": 90,
"description": "How old this individual is in years",
"type": "long"
},
"birthInfo": {
"display_name": "birth info",
"order": 5,
"description": "The year this individual was born",
"type": "double"
}
},
"required": [
"idValue",
"idType",
"eventTimestamp",
"countryCode",
"age"
],
"primary": "/age"
},
"status": "active",
"tags": [
"age",
"demo",
"demographic",
"demographics"
],
"updated_at": "2021-06-24T00:54:40.029056Z",
"write_mode": "append",
"subscription_id": "123e4567-e89b-12d3-a456-426614174000",
"last_snapshot_created_at": "2023-06-09T16:51:34.215Z",
"stats": {
"est_total_dataset_stored_files": 10,
"est_total_dataset_stored_records": 500059,
"est_total_dataset_stored_bytes": 4206454,
"active_dataset_stored_files": 10,
"active_dataset_stored_records": 500059,
"active_dataset_stored_bytes": 4206454,
"last_snapshot_added_files": 2,
"last_snapshot_added_records": 499999,
"last_snapshot_deleted_records": 0,
"last_snapshot_added_bytes": 4193500,
"last_snapshot_deleted_files": 0,
"last_snapshot_added_delete_files": 0,
"last_snapshot_removed_delete_files": 0,
"last_snapshot_removed_bytes": 0
}
}Datasets
Update a dataset's partition spec
Update the partition specification of the Iceberg table backing the dataset.
See:
POST
/
datasets
/
{dataset_id}
/
admin
/
partition-spec
Update a dataset's partition spec
curl --request POST \
--url https://api-dev.narrative.io/datasets/{dataset_id}/admin/partition-spec \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"updates": [
{
"type": "add",
"field": "id.value",
"transform": "bucket[128]"
},
{
"type": "remove",
"field": "__NARRATIVE_DEFAULT_PARTITION___bucket"
}
]
}
'import requests
url = "https://api-dev.narrative.io/datasets/{dataset_id}/admin/partition-spec"
payload = { "updates": [
{
"type": "add",
"field": "id.value",
"transform": "bucket[128]"
},
{
"type": "remove",
"field": "__NARRATIVE_DEFAULT_PARTITION___bucket"
}
] }
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
updates: [
{type: 'add', field: 'id.value', transform: 'bucket[128]'},
{type: 'remove', field: '__NARRATIVE_DEFAULT_PARTITION___bucket'}
]
})
};
fetch('https://api-dev.narrative.io/datasets/{dataset_id}/admin/partition-spec', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api-dev.narrative.io/datasets/{dataset_id}/admin/partition-spec",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'updates' => [
[
'type' => 'add',
'field' => 'id.value',
'transform' => 'bucket[128]'
],
[
'type' => 'remove',
'field' => '__NARRATIVE_DEFAULT_PARTITION___bucket'
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api-dev.narrative.io/datasets/{dataset_id}/admin/partition-spec"
payload := strings.NewReader("{\n \"updates\": [\n {\n \"type\": \"add\",\n \"field\": \"id.value\",\n \"transform\": \"bucket[128]\"\n },\n {\n \"type\": \"remove\",\n \"field\": \"__NARRATIVE_DEFAULT_PARTITION___bucket\"\n }\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api-dev.narrative.io/datasets/{dataset_id}/admin/partition-spec")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"updates\": [\n {\n \"type\": \"add\",\n \"field\": \"id.value\",\n \"transform\": \"bucket[128]\"\n },\n {\n \"type\": \"remove\",\n \"field\": \"__NARRATIVE_DEFAULT_PARTITION___bucket\"\n }\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api-dev.narrative.io/datasets/{dataset_id}/admin/partition-spec")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"updates\": [\n {\n \"type\": \"add\",\n \"field\": \"id.value\",\n \"transform\": \"bucket[128]\"\n },\n {\n \"type\": \"remove\",\n \"field\": \"__NARRATIVE_DEFAULT_PARTITION___bucket\"\n }\n ]\n}"
response = http.request(request)
puts response.read_body{
"id": 18923,
"company_id": 382,
"created_at": "2021-06-24T00:54:40.029056Z",
"data_plane": {
"type": "owned",
"id": "c0bed5c2-8af8-4f7a-9787-1ffc3def339b",
"company_id": 382,
"created_at": "2021-06-01T00:00:00Z",
"description": "The company's Snowflake account",
"display_name": "Snowflake (us-west-2)",
"external_id": "snowflake-prod",
"platform": {
"type": "platform_snowflake",
"account_locator": "ab12345",
"account_name": "acme",
"organization_name": "acmeorg",
"region": {
"type": "region_aws",
"id": "us-west-2"
}
},
"status": "active",
"tags": [],
"updated_at": "2021-06-01T00:00:00Z",
"collaborators": {
"participants": {
"type": "all"
},
"manage_compute_pools": {
"type": "all"
}
},
"compute_pools": []
},
"is_narrative_managed": true,
"retention_policy": {
"policies": [
{
"type": {
"kind": "snapshot_ttl",
"snapshot_retention_policy": {
"type": "retain_everything"
}
},
"enabled": true
}
]
},
"description": "Mobile identifiers tied to birth information provided as part of app registration.",
"name": "declared_age",
"display_name": "Declared Age",
"schema": {
"file_config": {
"type": "flat",
"delimiter": ",",
"escape": "\"",
"header": true,
"quote": "\""
},
"type": "object",
"properties": {
"idValue": {
"display_name": "id value",
"order": 0,
"approximate_cardinality": 100000,
"description": "ID values",
"type": "string"
},
"idType": {
"display_name": "id type",
"order": 1,
"approximate_cardinality": 1,
"type": "string",
"enum": [
"md5_email",
"sha256_email"
]
},
"eventTimestamp": {
"display_name": "event timestamp",
"order": 2,
"description": "When the observation was collected",
"type": "timestamptz",
"validations": [
"eventTimestamp < current_timestamp()"
]
},
"countryCode": {
"display_name": "country code",
"order": 3,
"approximate_cardinality": 6,
"description": "The country of residence of the individual",
"type": "string"
},
"age": {
"display_name": "age",
"order": 4,
"approximate_cardinality": 90,
"description": "How old this individual is in years",
"type": "long"
},
"birthInfo": {
"display_name": "birth info",
"order": 5,
"description": "The year this individual was born",
"type": "double"
}
},
"required": [
"idValue",
"idType",
"eventTimestamp",
"countryCode",
"age"
],
"primary": "/age"
},
"status": "active",
"tags": [
"age",
"demo",
"demographic",
"demographics"
],
"updated_at": "2021-06-24T00:54:40.029056Z",
"write_mode": "append",
"subscription_id": "123e4567-e89b-12d3-a456-426614174000",
"last_snapshot_created_at": "2023-06-09T16:51:34.215Z",
"stats": {
"est_total_dataset_stored_files": 10,
"est_total_dataset_stored_records": 500059,
"est_total_dataset_stored_bytes": 4206454,
"active_dataset_stored_files": 10,
"active_dataset_stored_records": 500059,
"active_dataset_stored_bytes": 4206454,
"last_snapshot_added_files": 2,
"last_snapshot_added_records": 499999,
"last_snapshot_deleted_records": 0,
"last_snapshot_added_bytes": 4193500,
"last_snapshot_deleted_files": 0,
"last_snapshot_added_delete_files": 0,
"last_snapshot_removed_delete_files": 0,
"last_snapshot_removed_bytes": 0
}
}Authorizations
Bearer authentication header of the form Bearer <token>, where <token> is your auth token.
Path Parameters
Unique identifier for a dataset.
Body
application/json
- Option 1
- Option 2
Show child attributes
Show child attributes
Response
200 - application/json
OK
A dataset.
The encoder drops nulls and empty collections, so an optional field is absent rather than null,
and tags and mappings are absent rather than [] when there are none. DatasetAdminResponse
composes this schema with allOf but encodes without that step, so on the admin endpoints the
same optional fields arrive as null. Either way they are optional, and both shapes validate.
Unique identifier for the dataset.
The ID of the company owning the dataset.
The data plane the dataset lives on.
- Option 1
- Option 2
Show child attributes
Show child attributes
Whether Narrative manages the dataset rather than the owning company.
ISO-8601 timestamp indicating when the dataset was created
The Dataset's name. Should be unique, contain only alpanum chars and underscores. Max length is 256.
The Dataset's name to display at UI.
A dataset's retention policies, plus the schedule they are enforced on. This is the shape every dataset response carries; legacy snapshot-only payloads are accepted on write and returned in this form.
Show child attributes
Show child attributes
Example:
{
"policies": [
{
"type": {
"kind": "table_ttl",
"clock": { "kind": "created_at" }
},
"enabled": true,
"interval": "P1D"
}
],
"schedule_config": {
"schedule": "@hourly",
"schedule_zone_id": "UTC",
"status": "active"
}
}
Schema describing a dataset's input file type and the kind of data the file contains. Note: when the file type is "flat" then the order of the properties describing the must be given in the order they appear in the input file.
Show child attributes
Show child attributes
The dataset status.
activemeans that data can be added to the dataset but the schema is locked in placependingmeans the dataset has been created, but the schema can still be altered and no data can be added to the datasetarchivedmeans the dataset has been retired and can no longer be read from or written to
Available options:
active, pending, archived How Narrative will treat new data uploaded for the dataset.
appendmeans data will be added to the dataset, this is what to choose if you incrementally update your dataset over time.overwritemeans that all existing data will be overwritten with new data.
Available options:
append, overwrite ISO-8601 timestamp indicating when the dataset was last updated.
The dataset's identifier in the system it came from. Absent when the dataset did not come from one.
The mappings from this dataset's fields to attributes. Absent when the dataset has none.
Show child attributes
Show child attributes
Compute pool configuration for the dataset.
Show child attributes
Show child attributes
A description of the data contained with the dataset.
Tags that describe the dataset.
Id of a subscription which writes output to the dataset
ISO-8601 timestamp indicating when the last snapshot was created.
Materialized-view configuration. Present only for materialized-view datasets.
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Metadata of Iceberg table backing the given Dataset encoded in spec-compliant JSON.
See:
Fields Narrative precomputes and stores in the table alongside the uploaded data — samples, indexes and mapped attribute values. Absent when the dataset has none.
Show child attributes
Show child attributes
Was this page helpful?