Create a dataset
curl --request POST \
--url https://api-dev.narrative.io/datasets \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"name": "sample_data",
"display_name": "sample data",
"description": "sample description",
"data_plane_id": "123e4567-e89b-12d3-a456-426614174000",
"external_id": "ext-dataset-001",
"is_narrative_managed": false,
"schema": {
"file_config": {
"type": "flat",
"delimiter": ",",
"escape": "\"",
"header": true,
"quote": "\""
},
"type": "object",
"properties": {
"idValue": {
"approximate_cardinality": 100000,
"description": "ID values",
"type": "string"
},
"idType": {
"approximate_cardinality": 1,
"type": "string",
"enum": [
"md5_email",
"sha256_email"
]
},
"eventTimestamp": {
"description": "When the observation was collected",
"type": "timestamptz",
"validations": [
"eventTimestamp < current_timestamp()"
]
},
"countryCode": {
"approximate_cardinality": 6,
"description": "The country of residence of the individual",
"type": "string"
},
"age": {
"approximate_cardinality": 90,
"description": "How old this individual is in years",
"type": "long"
},
"birthInfo": {
"description": "The year this individual was born",
"type": "double"
}
},
"required": [
"idValue",
"idType",
"eventTimestamp",
"countryCode",
"age"
],
"primary": "/age"
},
"tags": [
"age",
"demo",
"demographic",
"demographics"
],
"write_mode": "append",
"derive_metrics_config": true
}
'import requests
url = "https://api-dev.narrative.io/datasets"
payload = {
"name": "sample_data",
"display_name": "sample data",
"description": "sample description",
"data_plane_id": "123e4567-e89b-12d3-a456-426614174000",
"external_id": "ext-dataset-001",
"is_narrative_managed": False,
"schema": {
"file_config": {
"type": "flat",
"delimiter": ",",
"escape": "\"",
"header": True,
"quote": "\""
},
"type": "object",
"properties": {
"idValue": {
"approximate_cardinality": 100000,
"description": "ID values",
"type": "string"
},
"idType": {
"approximate_cardinality": 1,
"type": "string",
"enum": ["md5_email", "sha256_email"]
},
"eventTimestamp": {
"description": "When the observation was collected",
"type": "timestamptz",
"validations": ["eventTimestamp < current_timestamp()"]
},
"countryCode": {
"approximate_cardinality": 6,
"description": "The country of residence of the individual",
"type": "string"
},
"age": {
"approximate_cardinality": 90,
"description": "How old this individual is in years",
"type": "long"
},
"birthInfo": {
"description": "The year this individual was born",
"type": "double"
}
},
"required": ["idValue", "idType", "eventTimestamp", "countryCode", "age"],
"primary": "/age"
},
"tags": ["age", "demo", "demographic", "demographics"],
"write_mode": "append",
"derive_metrics_config": True
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
name: 'sample_data',
display_name: 'sample data',
description: 'sample description',
data_plane_id: '123e4567-e89b-12d3-a456-426614174000',
external_id: 'ext-dataset-001',
is_narrative_managed: false,
schema: {
file_config: {type: 'flat', delimiter: ',', escape: '"', header: true, quote: '"'},
type: 'object',
properties: {
idValue: {approximate_cardinality: 100000, description: 'ID values', type: 'string'},
idType: {
approximate_cardinality: 1,
type: 'string',
enum: ['md5_email', 'sha256_email']
},
eventTimestamp: {
description: 'When the observation was collected',
type: 'timestamptz',
validations: ['eventTimestamp < current_timestamp()']
},
countryCode: {
approximate_cardinality: 6,
description: 'The country of residence of the individual',
type: 'string'
},
age: {
approximate_cardinality: 90,
description: 'How old this individual is in years',
type: 'long'
},
birthInfo: {description: 'The year this individual was born', type: 'double'}
},
required: ['idValue', 'idType', 'eventTimestamp', 'countryCode', 'age'],
primary: '/age'
},
tags: ['age', 'demo', 'demographic', 'demographics'],
write_mode: 'append',
derive_metrics_config: true
})
};
fetch('https://api-dev.narrative.io/datasets', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api-dev.narrative.io/datasets",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'name' => 'sample_data',
'display_name' => 'sample data',
'description' => 'sample description',
'data_plane_id' => '123e4567-e89b-12d3-a456-426614174000',
'external_id' => 'ext-dataset-001',
'is_narrative_managed' => false,
'schema' => [
'file_config' => [
'type' => 'flat',
'delimiter' => ',',
'escape' => '"',
'header' => true,
'quote' => '"'
],
'type' => 'object',
'properties' => [
'idValue' => [
'approximate_cardinality' => 100000,
'description' => 'ID values',
'type' => 'string'
],
'idType' => [
'approximate_cardinality' => 1,
'type' => 'string',
'enum' => [
'md5_email',
'sha256_email'
]
],
'eventTimestamp' => [
'description' => 'When the observation was collected',
'type' => 'timestamptz',
'validations' => [
'eventTimestamp < current_timestamp()'
]
],
'countryCode' => [
'approximate_cardinality' => 6,
'description' => 'The country of residence of the individual',
'type' => 'string'
],
'age' => [
'approximate_cardinality' => 90,
'description' => 'How old this individual is in years',
'type' => 'long'
],
'birthInfo' => [
'description' => 'The year this individual was born',
'type' => 'double'
]
],
'required' => [
'idValue',
'idType',
'eventTimestamp',
'countryCode',
'age'
],
'primary' => '/age'
],
'tags' => [
'age',
'demo',
'demographic',
'demographics'
],
'write_mode' => 'append',
'derive_metrics_config' => true
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api-dev.narrative.io/datasets"
payload := strings.NewReader("{\n \"name\": \"sample_data\",\n \"display_name\": \"sample data\",\n \"description\": \"sample description\",\n \"data_plane_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"external_id\": \"ext-dataset-001\",\n \"is_narrative_managed\": false,\n \"schema\": {\n \"file_config\": {\n \"type\": \"flat\",\n \"delimiter\": \",\",\n \"escape\": \"\\\"\",\n \"header\": true,\n \"quote\": \"\\\"\"\n },\n \"type\": \"object\",\n \"properties\": {\n \"idValue\": {\n \"approximate_cardinality\": 100000,\n \"description\": \"ID values\",\n \"type\": \"string\"\n },\n \"idType\": {\n \"approximate_cardinality\": 1,\n \"type\": \"string\",\n \"enum\": [\n \"md5_email\",\n \"sha256_email\"\n ]\n },\n \"eventTimestamp\": {\n \"description\": \"When the observation was collected\",\n \"type\": \"timestamptz\",\n \"validations\": [\n \"eventTimestamp < current_timestamp()\"\n ]\n },\n \"countryCode\": {\n \"approximate_cardinality\": 6,\n \"description\": \"The country of residence of the individual\",\n \"type\": \"string\"\n },\n \"age\": {\n \"approximate_cardinality\": 90,\n \"description\": \"How old this individual is in years\",\n \"type\": \"long\"\n },\n \"birthInfo\": {\n \"description\": \"The year this individual was born\",\n \"type\": \"double\"\n }\n },\n \"required\": [\n \"idValue\",\n \"idType\",\n \"eventTimestamp\",\n \"countryCode\",\n \"age\"\n ],\n \"primary\": \"/age\"\n },\n \"tags\": [\n \"age\",\n \"demo\",\n \"demographic\",\n \"demographics\"\n ],\n \"write_mode\": \"append\",\n \"derive_metrics_config\": true\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api-dev.narrative.io/datasets")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"name\": \"sample_data\",\n \"display_name\": \"sample data\",\n \"description\": \"sample description\",\n \"data_plane_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"external_id\": \"ext-dataset-001\",\n \"is_narrative_managed\": false,\n \"schema\": {\n \"file_config\": {\n \"type\": \"flat\",\n \"delimiter\": \",\",\n \"escape\": \"\\\"\",\n \"header\": true,\n \"quote\": \"\\\"\"\n },\n \"type\": \"object\",\n \"properties\": {\n \"idValue\": {\n \"approximate_cardinality\": 100000,\n \"description\": \"ID values\",\n \"type\": \"string\"\n },\n \"idType\": {\n \"approximate_cardinality\": 1,\n \"type\": \"string\",\n \"enum\": [\n \"md5_email\",\n \"sha256_email\"\n ]\n },\n \"eventTimestamp\": {\n \"description\": \"When the observation was collected\",\n \"type\": \"timestamptz\",\n \"validations\": [\n \"eventTimestamp < current_timestamp()\"\n ]\n },\n \"countryCode\": {\n \"approximate_cardinality\": 6,\n \"description\": \"The country of residence of the individual\",\n \"type\": \"string\"\n },\n \"age\": {\n \"approximate_cardinality\": 90,\n \"description\": \"How old this individual is in years\",\n \"type\": \"long\"\n },\n \"birthInfo\": {\n \"description\": \"The year this individual was born\",\n \"type\": \"double\"\n }\n },\n \"required\": [\n \"idValue\",\n \"idType\",\n \"eventTimestamp\",\n \"countryCode\",\n \"age\"\n ],\n \"primary\": \"/age\"\n },\n \"tags\": [\n \"age\",\n \"demo\",\n \"demographic\",\n \"demographics\"\n ],\n \"write_mode\": \"append\",\n \"derive_metrics_config\": true\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api-dev.narrative.io/datasets")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"name\": \"sample_data\",\n \"display_name\": \"sample data\",\n \"description\": \"sample description\",\n \"data_plane_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"external_id\": \"ext-dataset-001\",\n \"is_narrative_managed\": false,\n \"schema\": {\n \"file_config\": {\n \"type\": \"flat\",\n \"delimiter\": \",\",\n \"escape\": \"\\\"\",\n \"header\": true,\n \"quote\": \"\\\"\"\n },\n \"type\": \"object\",\n \"properties\": {\n \"idValue\": {\n \"approximate_cardinality\": 100000,\n \"description\": \"ID values\",\n \"type\": \"string\"\n },\n \"idType\": {\n \"approximate_cardinality\": 1,\n \"type\": \"string\",\n \"enum\": [\n \"md5_email\",\n \"sha256_email\"\n ]\n },\n \"eventTimestamp\": {\n \"description\": \"When the observation was collected\",\n \"type\": \"timestamptz\",\n \"validations\": [\n \"eventTimestamp < current_timestamp()\"\n ]\n },\n \"countryCode\": {\n \"approximate_cardinality\": 6,\n \"description\": \"The country of residence of the individual\",\n \"type\": \"string\"\n },\n \"age\": {\n \"approximate_cardinality\": 90,\n \"description\": \"How old this individual is in years\",\n \"type\": \"long\"\n },\n \"birthInfo\": {\n \"description\": \"The year this individual was born\",\n \"type\": \"double\"\n }\n },\n \"required\": [\n \"idValue\",\n \"idType\",\n \"eventTimestamp\",\n \"countryCode\",\n \"age\"\n ],\n \"primary\": \"/age\"\n },\n \"tags\": [\n \"age\",\n \"demo\",\n \"demographic\",\n \"demographics\"\n ],\n \"write_mode\": \"append\",\n \"derive_metrics_config\": true\n}"
response = http.request(request)
puts response.read_body{
"id": 18923,
"company_id": 382,
"created_at": "2021-06-24T00:54:40.029056Z",
"description": "Mobile identifiers tied to birth information provided as part of app registration.",
"name": "declared_age",
"display_name": "Declared Age",
"schema": {
"file_config": {
"type": "flat",
"delimiter": ",",
"escape": "\"",
"header": true,
"quote": "\""
},
"type": "object",
"properties": {
"idValue": {
"display_name": "id value",
"order": 0,
"approximate_cardinality": 100000,
"description": "ID values",
"type": "string"
},
"idType": {
"display_name": "id type",
"order": 1,
"approximate_cardinality": 1,
"type": "string",
"enum": [
"md5_email",
"sha256_email"
]
},
"eventTimestamp": {
"display_name": "event timestamp",
"order": 2,
"description": "When the observation was collected",
"type": "timestamptz",
"validations": [
"eventTimestamp < current_timestamp()"
]
},
"countryCode": {
"display_name": "country code",
"order": 3,
"approximate_cardinality": 6,
"description": "The country of residence of the individual",
"type": "string"
},
"age": {
"display_name": "age",
"order": 4,
"approximate_cardinality": 90,
"description": "How old this individual is in years",
"type": "long"
},
"birthInfo": {
"display_name": "birth info",
"order": 5,
"description": "The year this individual was born",
"type": "double"
}
},
"required": [
"idValue",
"idType",
"eventTimestamp",
"countryCode",
"age"
],
"primary": "/age"
},
"status": "active",
"tags": [
"age",
"demo",
"demographic",
"demographics"
],
"updated_at": "2021-06-24T00:54:40.029056Z",
"write_mode": "append",
"subscription_id": "123e4567-e89b-12d3-a456-426614174000",
"last_snapshot_created_at": "2023-06-09T16:51:34.215Z",
"stats": {
"est_total_dataset_stored_files": 10,
"est_total_dataset_stored_records": 500059,
"est_total_dataset_stored_bytes": 4206454,
"active_dataset_stored_files": 10,
"active_dataset_stored_records": 500059,
"active_dataset_stored_bytes": 4206454,
"last_snapshot_added_files": 2,
"last_snapshot_added_records": 499999,
"last_snapshot_deleted_records": 0,
"last_snapshot_added_bytes": 4193500,
"last_snapshot_deleted_files": 0,
"last_snapshot_added_delete_files": 0,
"last_snapshot_removed_delete_files": 0,
"last_snapshot_removed_bytes": 0
}
}{
"error": "Unauthorized",
"error_description": "You are not authorized to use this endpoint."
}Datasets
Create a dataset
Create or register a new dataset.
Newly created datasets are in the pending status and require activation.
To register an empty dataset in Snowflake Dataplane:
If the request has data_plane_id defined but external_id is not provided, it will create an empty dataset with the specified schema in pending state.
To trigger creation of such dataset in Snowflake Dataplane, it should call PUT datasets/<dataset_id>/activate.
POST
/
datasets
Create a dataset
curl --request POST \
--url https://api-dev.narrative.io/datasets \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"name": "sample_data",
"display_name": "sample data",
"description": "sample description",
"data_plane_id": "123e4567-e89b-12d3-a456-426614174000",
"external_id": "ext-dataset-001",
"is_narrative_managed": false,
"schema": {
"file_config": {
"type": "flat",
"delimiter": ",",
"escape": "\"",
"header": true,
"quote": "\""
},
"type": "object",
"properties": {
"idValue": {
"approximate_cardinality": 100000,
"description": "ID values",
"type": "string"
},
"idType": {
"approximate_cardinality": 1,
"type": "string",
"enum": [
"md5_email",
"sha256_email"
]
},
"eventTimestamp": {
"description": "When the observation was collected",
"type": "timestamptz",
"validations": [
"eventTimestamp < current_timestamp()"
]
},
"countryCode": {
"approximate_cardinality": 6,
"description": "The country of residence of the individual",
"type": "string"
},
"age": {
"approximate_cardinality": 90,
"description": "How old this individual is in years",
"type": "long"
},
"birthInfo": {
"description": "The year this individual was born",
"type": "double"
}
},
"required": [
"idValue",
"idType",
"eventTimestamp",
"countryCode",
"age"
],
"primary": "/age"
},
"tags": [
"age",
"demo",
"demographic",
"demographics"
],
"write_mode": "append",
"derive_metrics_config": true
}
'import requests
url = "https://api-dev.narrative.io/datasets"
payload = {
"name": "sample_data",
"display_name": "sample data",
"description": "sample description",
"data_plane_id": "123e4567-e89b-12d3-a456-426614174000",
"external_id": "ext-dataset-001",
"is_narrative_managed": False,
"schema": {
"file_config": {
"type": "flat",
"delimiter": ",",
"escape": "\"",
"header": True,
"quote": "\""
},
"type": "object",
"properties": {
"idValue": {
"approximate_cardinality": 100000,
"description": "ID values",
"type": "string"
},
"idType": {
"approximate_cardinality": 1,
"type": "string",
"enum": ["md5_email", "sha256_email"]
},
"eventTimestamp": {
"description": "When the observation was collected",
"type": "timestamptz",
"validations": ["eventTimestamp < current_timestamp()"]
},
"countryCode": {
"approximate_cardinality": 6,
"description": "The country of residence of the individual",
"type": "string"
},
"age": {
"approximate_cardinality": 90,
"description": "How old this individual is in years",
"type": "long"
},
"birthInfo": {
"description": "The year this individual was born",
"type": "double"
}
},
"required": ["idValue", "idType", "eventTimestamp", "countryCode", "age"],
"primary": "/age"
},
"tags": ["age", "demo", "demographic", "demographics"],
"write_mode": "append",
"derive_metrics_config": True
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
name: 'sample_data',
display_name: 'sample data',
description: 'sample description',
data_plane_id: '123e4567-e89b-12d3-a456-426614174000',
external_id: 'ext-dataset-001',
is_narrative_managed: false,
schema: {
file_config: {type: 'flat', delimiter: ',', escape: '"', header: true, quote: '"'},
type: 'object',
properties: {
idValue: {approximate_cardinality: 100000, description: 'ID values', type: 'string'},
idType: {
approximate_cardinality: 1,
type: 'string',
enum: ['md5_email', 'sha256_email']
},
eventTimestamp: {
description: 'When the observation was collected',
type: 'timestamptz',
validations: ['eventTimestamp < current_timestamp()']
},
countryCode: {
approximate_cardinality: 6,
description: 'The country of residence of the individual',
type: 'string'
},
age: {
approximate_cardinality: 90,
description: 'How old this individual is in years',
type: 'long'
},
birthInfo: {description: 'The year this individual was born', type: 'double'}
},
required: ['idValue', 'idType', 'eventTimestamp', 'countryCode', 'age'],
primary: '/age'
},
tags: ['age', 'demo', 'demographic', 'demographics'],
write_mode: 'append',
derive_metrics_config: true
})
};
fetch('https://api-dev.narrative.io/datasets', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api-dev.narrative.io/datasets",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'name' => 'sample_data',
'display_name' => 'sample data',
'description' => 'sample description',
'data_plane_id' => '123e4567-e89b-12d3-a456-426614174000',
'external_id' => 'ext-dataset-001',
'is_narrative_managed' => false,
'schema' => [
'file_config' => [
'type' => 'flat',
'delimiter' => ',',
'escape' => '"',
'header' => true,
'quote' => '"'
],
'type' => 'object',
'properties' => [
'idValue' => [
'approximate_cardinality' => 100000,
'description' => 'ID values',
'type' => 'string'
],
'idType' => [
'approximate_cardinality' => 1,
'type' => 'string',
'enum' => [
'md5_email',
'sha256_email'
]
],
'eventTimestamp' => [
'description' => 'When the observation was collected',
'type' => 'timestamptz',
'validations' => [
'eventTimestamp < current_timestamp()'
]
],
'countryCode' => [
'approximate_cardinality' => 6,
'description' => 'The country of residence of the individual',
'type' => 'string'
],
'age' => [
'approximate_cardinality' => 90,
'description' => 'How old this individual is in years',
'type' => 'long'
],
'birthInfo' => [
'description' => 'The year this individual was born',
'type' => 'double'
]
],
'required' => [
'idValue',
'idType',
'eventTimestamp',
'countryCode',
'age'
],
'primary' => '/age'
],
'tags' => [
'age',
'demo',
'demographic',
'demographics'
],
'write_mode' => 'append',
'derive_metrics_config' => true
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api-dev.narrative.io/datasets"
payload := strings.NewReader("{\n \"name\": \"sample_data\",\n \"display_name\": \"sample data\",\n \"description\": \"sample description\",\n \"data_plane_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"external_id\": \"ext-dataset-001\",\n \"is_narrative_managed\": false,\n \"schema\": {\n \"file_config\": {\n \"type\": \"flat\",\n \"delimiter\": \",\",\n \"escape\": \"\\\"\",\n \"header\": true,\n \"quote\": \"\\\"\"\n },\n \"type\": \"object\",\n \"properties\": {\n \"idValue\": {\n \"approximate_cardinality\": 100000,\n \"description\": \"ID values\",\n \"type\": \"string\"\n },\n \"idType\": {\n \"approximate_cardinality\": 1,\n \"type\": \"string\",\n \"enum\": [\n \"md5_email\",\n \"sha256_email\"\n ]\n },\n \"eventTimestamp\": {\n \"description\": \"When the observation was collected\",\n \"type\": \"timestamptz\",\n \"validations\": [\n \"eventTimestamp < current_timestamp()\"\n ]\n },\n \"countryCode\": {\n \"approximate_cardinality\": 6,\n \"description\": \"The country of residence of the individual\",\n \"type\": \"string\"\n },\n \"age\": {\n \"approximate_cardinality\": 90,\n \"description\": \"How old this individual is in years\",\n \"type\": \"long\"\n },\n \"birthInfo\": {\n \"description\": \"The year this individual was born\",\n \"type\": \"double\"\n }\n },\n \"required\": [\n \"idValue\",\n \"idType\",\n \"eventTimestamp\",\n \"countryCode\",\n \"age\"\n ],\n \"primary\": \"/age\"\n },\n \"tags\": [\n \"age\",\n \"demo\",\n \"demographic\",\n \"demographics\"\n ],\n \"write_mode\": \"append\",\n \"derive_metrics_config\": true\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api-dev.narrative.io/datasets")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"name\": \"sample_data\",\n \"display_name\": \"sample data\",\n \"description\": \"sample description\",\n \"data_plane_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"external_id\": \"ext-dataset-001\",\n \"is_narrative_managed\": false,\n \"schema\": {\n \"file_config\": {\n \"type\": \"flat\",\n \"delimiter\": \",\",\n \"escape\": \"\\\"\",\n \"header\": true,\n \"quote\": \"\\\"\"\n },\n \"type\": \"object\",\n \"properties\": {\n \"idValue\": {\n \"approximate_cardinality\": 100000,\n \"description\": \"ID values\",\n \"type\": \"string\"\n },\n \"idType\": {\n \"approximate_cardinality\": 1,\n \"type\": \"string\",\n \"enum\": [\n \"md5_email\",\n \"sha256_email\"\n ]\n },\n \"eventTimestamp\": {\n \"description\": \"When the observation was collected\",\n \"type\": \"timestamptz\",\n \"validations\": [\n \"eventTimestamp < current_timestamp()\"\n ]\n },\n \"countryCode\": {\n \"approximate_cardinality\": 6,\n \"description\": \"The country of residence of the individual\",\n \"type\": \"string\"\n },\n \"age\": {\n \"approximate_cardinality\": 90,\n \"description\": \"How old this individual is in years\",\n \"type\": \"long\"\n },\n \"birthInfo\": {\n \"description\": \"The year this individual was born\",\n \"type\": \"double\"\n }\n },\n \"required\": [\n \"idValue\",\n \"idType\",\n \"eventTimestamp\",\n \"countryCode\",\n \"age\"\n ],\n \"primary\": \"/age\"\n },\n \"tags\": [\n \"age\",\n \"demo\",\n \"demographic\",\n \"demographics\"\n ],\n \"write_mode\": \"append\",\n \"derive_metrics_config\": true\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api-dev.narrative.io/datasets")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"name\": \"sample_data\",\n \"display_name\": \"sample data\",\n \"description\": \"sample description\",\n \"data_plane_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"external_id\": \"ext-dataset-001\",\n \"is_narrative_managed\": false,\n \"schema\": {\n \"file_config\": {\n \"type\": \"flat\",\n \"delimiter\": \",\",\n \"escape\": \"\\\"\",\n \"header\": true,\n \"quote\": \"\\\"\"\n },\n \"type\": \"object\",\n \"properties\": {\n \"idValue\": {\n \"approximate_cardinality\": 100000,\n \"description\": \"ID values\",\n \"type\": \"string\"\n },\n \"idType\": {\n \"approximate_cardinality\": 1,\n \"type\": \"string\",\n \"enum\": [\n \"md5_email\",\n \"sha256_email\"\n ]\n },\n \"eventTimestamp\": {\n \"description\": \"When the observation was collected\",\n \"type\": \"timestamptz\",\n \"validations\": [\n \"eventTimestamp < current_timestamp()\"\n ]\n },\n \"countryCode\": {\n \"approximate_cardinality\": 6,\n \"description\": \"The country of residence of the individual\",\n \"type\": \"string\"\n },\n \"age\": {\n \"approximate_cardinality\": 90,\n \"description\": \"How old this individual is in years\",\n \"type\": \"long\"\n },\n \"birthInfo\": {\n \"description\": \"The year this individual was born\",\n \"type\": \"double\"\n }\n },\n \"required\": [\n \"idValue\",\n \"idType\",\n \"eventTimestamp\",\n \"countryCode\",\n \"age\"\n ],\n \"primary\": \"/age\"\n },\n \"tags\": [\n \"age\",\n \"demo\",\n \"demographic\",\n \"demographics\"\n ],\n \"write_mode\": \"append\",\n \"derive_metrics_config\": true\n}"
response = http.request(request)
puts response.read_body{
"id": 18923,
"company_id": 382,
"created_at": "2021-06-24T00:54:40.029056Z",
"description": "Mobile identifiers tied to birth information provided as part of app registration.",
"name": "declared_age",
"display_name": "Declared Age",
"schema": {
"file_config": {
"type": "flat",
"delimiter": ",",
"escape": "\"",
"header": true,
"quote": "\""
},
"type": "object",
"properties": {
"idValue": {
"display_name": "id value",
"order": 0,
"approximate_cardinality": 100000,
"description": "ID values",
"type": "string"
},
"idType": {
"display_name": "id type",
"order": 1,
"approximate_cardinality": 1,
"type": "string",
"enum": [
"md5_email",
"sha256_email"
]
},
"eventTimestamp": {
"display_name": "event timestamp",
"order": 2,
"description": "When the observation was collected",
"type": "timestamptz",
"validations": [
"eventTimestamp < current_timestamp()"
]
},
"countryCode": {
"display_name": "country code",
"order": 3,
"approximate_cardinality": 6,
"description": "The country of residence of the individual",
"type": "string"
},
"age": {
"display_name": "age",
"order": 4,
"approximate_cardinality": 90,
"description": "How old this individual is in years",
"type": "long"
},
"birthInfo": {
"display_name": "birth info",
"order": 5,
"description": "The year this individual was born",
"type": "double"
}
},
"required": [
"idValue",
"idType",
"eventTimestamp",
"countryCode",
"age"
],
"primary": "/age"
},
"status": "active",
"tags": [
"age",
"demo",
"demographic",
"demographics"
],
"updated_at": "2021-06-24T00:54:40.029056Z",
"write_mode": "append",
"subscription_id": "123e4567-e89b-12d3-a456-426614174000",
"last_snapshot_created_at": "2023-06-09T16:51:34.215Z",
"stats": {
"est_total_dataset_stored_files": 10,
"est_total_dataset_stored_records": 500059,
"est_total_dataset_stored_bytes": 4206454,
"active_dataset_stored_files": 10,
"active_dataset_stored_records": 500059,
"active_dataset_stored_bytes": 4206454,
"last_snapshot_added_files": 2,
"last_snapshot_added_records": 499999,
"last_snapshot_deleted_records": 0,
"last_snapshot_added_bytes": 4193500,
"last_snapshot_deleted_files": 0,
"last_snapshot_added_delete_files": 0,
"last_snapshot_removed_delete_files": 0,
"last_snapshot_removed_bytes": 0
}
}{
"error": "Unauthorized",
"error_description": "You are not authorized to use this endpoint."
}Authorizations
Bearer authentication header of the form Bearer <token>, where <token> is your auth token.
Body
application/json
The Dataset's name. Should be unique, contain only alpanum chars and underscores. Max length is 256.
Schema describing a dataset's input file type and the kind of data the file contains. Note: when the file type is "flat" then the order of the properties describing the must be given in the order they appear in the input file.
Show child attributes
Show child attributes
The Dataset's name to display at UI.
A description of the data contained with the dataset.
Optional data plane identifier, must be provided when registering dataset in external dataplane.
Optional external identifier, must be excluded when registering dataset in external dataplane
Whether the dataset is managed by Narrative
Tags that describe the dataset.
How Narrative will treat new data uploaded for the dataset.
appendmeans data will be added to the dataset, this is what to choose if you incrementally update your dataset over time.overwritemeans that all existing data will be overwritten with new data.
Available options:
append, overwrite A flag to enable derivation of the config.
Response
OK
Unique identifier for the dataset.
The ID of the company owning the dataset.
ISO-8601 timestamp indicating when the dataset was created
The Dataset's name. Should be unique, contain only alpanum chars and underscores. Max length is 256.
The Dataset's name to display at UI.
Backwards-compatible legacy payloads. Interpreted as snapshot-level retention on Iceberg. Returned response will be unified v1 shape with compat.mode=legacy.
- Option 1
- Option 2
- Option 3
- Option 4
Show child attributes
Show child attributes
Example:
{ "type": "expire_everything" }
Schema describing a dataset's input file type and the kind of data the file contains. Note: when the file type is "flat" then the order of the properties describing the must be given in the order they appear in the input file.
Show child attributes
Show child attributes
The dataset status.
activemeans that data can be added to the dataset but the schema is locked in placependingmeans the dataset has been created, but the schema can still be altered and no data can be added to the dataset
Available options:
active, pending Tags that describe the dataset.
How Narrative will treat new data uploaded for the dataset.
appendmeans data will be added to the dataset, this is what to choose if you incrementally update your dataset over time.overwritemeans that all existing data will be overwritten with new data.
Available options:
append, overwrite ISO-8601 timestamp indicating when the dataset was last updated.
Compute pool configuration for the dataset.
Show child attributes
Show child attributes
A description of the data contained with the dataset.
Id of a subscription which writes output to the dataset
ISO-8601 timestamp indicating when the last snapshot was created.
Materialized-view configuration. Present only for materialized-view datasets.
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Was this page helpful?
⌘I