Create a dataset
curl --request POST \
--url https://api-dev.narrative.io/datasets \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"name": "sample_data",
"display_name": "sample data",
"description": "sample description",
"data_plane_id": "123e4567-e89b-12d3-a456-426614174000",
"external_id": "ext-dataset-001",
"is_narrative_managed": false,
"schema": {
"file_config": {
"type": "flat",
"delimiter": ",",
"escape": "\"",
"header": true,
"quote": "\""
},
"type": "object",
"properties": {
"idValue": {
"approximate_cardinality": 100000,
"description": "ID values",
"type": "string"
},
"idType": {
"approximate_cardinality": 1,
"type": "string",
"enum": [
"md5_email",
"sha256_email"
]
},
"eventTimestamp": {
"description": "When the observation was collected",
"type": "timestamptz",
"validations": [
"eventTimestamp < current_timestamp()"
]
},
"countryCode": {
"approximate_cardinality": 6,
"description": "The country of residence of the individual",
"type": "string"
},
"age": {
"approximate_cardinality": 90,
"description": "How old this individual is in years",
"type": "long"
},
"birthInfo": {
"description": "The year this individual was born",
"type": "double"
}
},
"required": [
"idValue",
"idType",
"eventTimestamp",
"countryCode",
"age"
],
"primary": "/age"
},
"tags": [
"age",
"demo",
"demographic",
"demographics"
],
"write_mode": "append",
"derive_metrics_config": true
}
'import requests
url = "https://api-dev.narrative.io/datasets"
payload = {
"name": "sample_data",
"display_name": "sample data",
"description": "sample description",
"data_plane_id": "123e4567-e89b-12d3-a456-426614174000",
"external_id": "ext-dataset-001",
"is_narrative_managed": False,
"schema": {
"file_config": {
"type": "flat",
"delimiter": ",",
"escape": "\"",
"header": True,
"quote": "\""
},
"type": "object",
"properties": {
"idValue": {
"approximate_cardinality": 100000,
"description": "ID values",
"type": "string"
},
"idType": {
"approximate_cardinality": 1,
"type": "string",
"enum": ["md5_email", "sha256_email"]
},
"eventTimestamp": {
"description": "When the observation was collected",
"type": "timestamptz",
"validations": ["eventTimestamp < current_timestamp()"]
},
"countryCode": {
"approximate_cardinality": 6,
"description": "The country of residence of the individual",
"type": "string"
},
"age": {
"approximate_cardinality": 90,
"description": "How old this individual is in years",
"type": "long"
},
"birthInfo": {
"description": "The year this individual was born",
"type": "double"
}
},
"required": ["idValue", "idType", "eventTimestamp", "countryCode", "age"],
"primary": "/age"
},
"tags": ["age", "demo", "demographic", "demographics"],
"write_mode": "append",
"derive_metrics_config": True
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
name: 'sample_data',
display_name: 'sample data',
description: 'sample description',
data_plane_id: '123e4567-e89b-12d3-a456-426614174000',
external_id: 'ext-dataset-001',
is_narrative_managed: false,
schema: {
file_config: {type: 'flat', delimiter: ',', escape: '"', header: true, quote: '"'},
type: 'object',
properties: {
idValue: {approximate_cardinality: 100000, description: 'ID values', type: 'string'},
idType: {
approximate_cardinality: 1,
type: 'string',
enum: ['md5_email', 'sha256_email']
},
eventTimestamp: {
description: 'When the observation was collected',
type: 'timestamptz',
validations: ['eventTimestamp < current_timestamp()']
},
countryCode: {
approximate_cardinality: 6,
description: 'The country of residence of the individual',
type: 'string'
},
age: {
approximate_cardinality: 90,
description: 'How old this individual is in years',
type: 'long'
},
birthInfo: {description: 'The year this individual was born', type: 'double'}
},
required: ['idValue', 'idType', 'eventTimestamp', 'countryCode', 'age'],
primary: '/age'
},
tags: ['age', 'demo', 'demographic', 'demographics'],
write_mode: 'append',
derive_metrics_config: true
})
};
fetch('https://api-dev.narrative.io/datasets', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api-dev.narrative.io/datasets",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'name' => 'sample_data',
'display_name' => 'sample data',
'description' => 'sample description',
'data_plane_id' => '123e4567-e89b-12d3-a456-426614174000',
'external_id' => 'ext-dataset-001',
'is_narrative_managed' => false,
'schema' => [
'file_config' => [
'type' => 'flat',
'delimiter' => ',',
'escape' => '"',
'header' => true,
'quote' => '"'
],
'type' => 'object',
'properties' => [
'idValue' => [
'approximate_cardinality' => 100000,
'description' => 'ID values',
'type' => 'string'
],
'idType' => [
'approximate_cardinality' => 1,
'type' => 'string',
'enum' => [
'md5_email',
'sha256_email'
]
],
'eventTimestamp' => [
'description' => 'When the observation was collected',
'type' => 'timestamptz',
'validations' => [
'eventTimestamp < current_timestamp()'
]
],
'countryCode' => [
'approximate_cardinality' => 6,
'description' => 'The country of residence of the individual',
'type' => 'string'
],
'age' => [
'approximate_cardinality' => 90,
'description' => 'How old this individual is in years',
'type' => 'long'
],
'birthInfo' => [
'description' => 'The year this individual was born',
'type' => 'double'
]
],
'required' => [
'idValue',
'idType',
'eventTimestamp',
'countryCode',
'age'
],
'primary' => '/age'
],
'tags' => [
'age',
'demo',
'demographic',
'demographics'
],
'write_mode' => 'append',
'derive_metrics_config' => true
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api-dev.narrative.io/datasets"
payload := strings.NewReader("{\n \"name\": \"sample_data\",\n \"display_name\": \"sample data\",\n \"description\": \"sample description\",\n \"data_plane_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"external_id\": \"ext-dataset-001\",\n \"is_narrative_managed\": false,\n \"schema\": {\n \"file_config\": {\n \"type\": \"flat\",\n \"delimiter\": \",\",\n \"escape\": \"\\\"\",\n \"header\": true,\n \"quote\": \"\\\"\"\n },\n \"type\": \"object\",\n \"properties\": {\n \"idValue\": {\n \"approximate_cardinality\": 100000,\n \"description\": \"ID values\",\n \"type\": \"string\"\n },\n \"idType\": {\n \"approximate_cardinality\": 1,\n \"type\": \"string\",\n \"enum\": [\n \"md5_email\",\n \"sha256_email\"\n ]\n },\n \"eventTimestamp\": {\n \"description\": \"When the observation was collected\",\n \"type\": \"timestamptz\",\n \"validations\": [\n \"eventTimestamp < current_timestamp()\"\n ]\n },\n \"countryCode\": {\n \"approximate_cardinality\": 6,\n \"description\": \"The country of residence of the individual\",\n \"type\": \"string\"\n },\n \"age\": {\n \"approximate_cardinality\": 90,\n \"description\": \"How old this individual is in years\",\n \"type\": \"long\"\n },\n \"birthInfo\": {\n \"description\": \"The year this individual was born\",\n \"type\": \"double\"\n }\n },\n \"required\": [\n \"idValue\",\n \"idType\",\n \"eventTimestamp\",\n \"countryCode\",\n \"age\"\n ],\n \"primary\": \"/age\"\n },\n \"tags\": [\n \"age\",\n \"demo\",\n \"demographic\",\n \"demographics\"\n ],\n \"write_mode\": \"append\",\n \"derive_metrics_config\": true\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api-dev.narrative.io/datasets")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"name\": \"sample_data\",\n \"display_name\": \"sample data\",\n \"description\": \"sample description\",\n \"data_plane_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"external_id\": \"ext-dataset-001\",\n \"is_narrative_managed\": false,\n \"schema\": {\n \"file_config\": {\n \"type\": \"flat\",\n \"delimiter\": \",\",\n \"escape\": \"\\\"\",\n \"header\": true,\n \"quote\": \"\\\"\"\n },\n \"type\": \"object\",\n \"properties\": {\n \"idValue\": {\n \"approximate_cardinality\": 100000,\n \"description\": \"ID values\",\n \"type\": \"string\"\n },\n \"idType\": {\n \"approximate_cardinality\": 1,\n \"type\": \"string\",\n \"enum\": [\n \"md5_email\",\n \"sha256_email\"\n ]\n },\n \"eventTimestamp\": {\n \"description\": \"When the observation was collected\",\n \"type\": \"timestamptz\",\n \"validations\": [\n \"eventTimestamp < current_timestamp()\"\n ]\n },\n \"countryCode\": {\n \"approximate_cardinality\": 6,\n \"description\": \"The country of residence of the individual\",\n \"type\": \"string\"\n },\n \"age\": {\n \"approximate_cardinality\": 90,\n \"description\": \"How old this individual is in years\",\n \"type\": \"long\"\n },\n \"birthInfo\": {\n \"description\": \"The year this individual was born\",\n \"type\": \"double\"\n }\n },\n \"required\": [\n \"idValue\",\n \"idType\",\n \"eventTimestamp\",\n \"countryCode\",\n \"age\"\n ],\n \"primary\": \"/age\"\n },\n \"tags\": [\n \"age\",\n \"demo\",\n \"demographic\",\n \"demographics\"\n ],\n \"write_mode\": \"append\",\n \"derive_metrics_config\": true\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api-dev.narrative.io/datasets")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"name\": \"sample_data\",\n \"display_name\": \"sample data\",\n \"description\": \"sample description\",\n \"data_plane_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"external_id\": \"ext-dataset-001\",\n \"is_narrative_managed\": false,\n \"schema\": {\n \"file_config\": {\n \"type\": \"flat\",\n \"delimiter\": \",\",\n \"escape\": \"\\\"\",\n \"header\": true,\n \"quote\": \"\\\"\"\n },\n \"type\": \"object\",\n \"properties\": {\n \"idValue\": {\n \"approximate_cardinality\": 100000,\n \"description\": \"ID values\",\n \"type\": \"string\"\n },\n \"idType\": {\n \"approximate_cardinality\": 1,\n \"type\": \"string\",\n \"enum\": [\n \"md5_email\",\n \"sha256_email\"\n ]\n },\n \"eventTimestamp\": {\n \"description\": \"When the observation was collected\",\n \"type\": \"timestamptz\",\n \"validations\": [\n \"eventTimestamp < current_timestamp()\"\n ]\n },\n \"countryCode\": {\n \"approximate_cardinality\": 6,\n \"description\": \"The country of residence of the individual\",\n \"type\": \"string\"\n },\n \"age\": {\n \"approximate_cardinality\": 90,\n \"description\": \"How old this individual is in years\",\n \"type\": \"long\"\n },\n \"birthInfo\": {\n \"description\": \"The year this individual was born\",\n \"type\": \"double\"\n }\n },\n \"required\": [\n \"idValue\",\n \"idType\",\n \"eventTimestamp\",\n \"countryCode\",\n \"age\"\n ],\n \"primary\": \"/age\"\n },\n \"tags\": [\n \"age\",\n \"demo\",\n \"demographic\",\n \"demographics\"\n ],\n \"write_mode\": \"append\",\n \"derive_metrics_config\": true\n}"
response = http.request(request)
puts response.read_body{
"id": 18923,
"company_id": 382,
"created_at": "2021-06-24T00:54:40.029056Z",
"description": "Mobile identifiers tied to birth information provided as part of app registration.",
"name": "declared_age",
"display_name": "Declared Age",
"schema": {
"file_config": {
"type": "flat",
"delimiter": ",",
"escape": "\"",
"header": true,
"quote": "\""
},
"type": "object",
"properties": {
"idValue": {
"display_name": "id value",
"order": 0,
"approximate_cardinality": 100000,
"description": "ID values",
"type": "string"
},
"idType": {
"display_name": "id type",
"order": 1,
"approximate_cardinality": 1,
"type": "string",
"enum": [
"md5_email",
"sha256_email"
]
},
"eventTimestamp": {
"display_name": "event timestamp",
"order": 2,
"description": "When the observation was collected",
"type": "timestamptz",
"validations": [
"eventTimestamp < current_timestamp()"
]
},
"countryCode": {
"display_name": "country code",
"order": 3,
"approximate_cardinality": 6,
"description": "The country of residence of the individual",
"type": "string"
},
"age": {
"display_name": "age",
"order": 4,
"approximate_cardinality": 90,
"description": "How old this individual is in years",
"type": "long"
},
"birthInfo": {
"display_name": "birth info",
"order": 5,
"description": "The year this individual was born",
"type": "double"
}
},
"required": [
"idValue",
"idType",
"eventTimestamp",
"countryCode",
"age"
],
"primary": "/age"
},
"status": "active",
"tags": [
"age",
"demo",
"demographic",
"demographics"
],
"updated_at": "2021-06-24T00:54:40.029056Z",
"write_mode": "append",
"subscription_id": "123e4567-e89b-12d3-a456-426614174000",
"last_snapshot_created_at": "2023-06-09T16:51:34.215Z",
"stats": {
"est_total_dataset_stored_files": 10,
"est_total_dataset_stored_records": 500059,
"est_total_dataset_stored_bytes": 4206454,
"active_dataset_stored_files": 10,
"active_dataset_stored_records": 500059,
"active_dataset_stored_bytes": 4206454,
"last_snapshot_added_files": 2,
"last_snapshot_added_records": 499999,
"last_snapshot_deleted_records": 0,
"last_snapshot_added_bytes": 4193500,
"last_snapshot_deleted_files": 0,
"last_snapshot_added_delete_files": 0,
"last_snapshot_removed_delete_files": 0,
"last_snapshot_removed_bytes": 0
}
}{
"error": "Unauthorized",
"error_description": "You are not authorized to use this endpoint."
}Datasets
Create a dataset
Create or register a new dataset.
Newly created datasets are in the pending status and require activation.
To register an empty dataset in Snowflake Dataplane:
If the request has data_plane_id defined but external_id is not provided, it will create an empty dataset with the specified schema in pending state.
To trigger creation of such dataset in Snowflake Dataplane, it should call PUT datasets/<dataset_id>/activate.
POST
/
datasets
Create a dataset
curl --request POST \
--url https://api-dev.narrative.io/datasets \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"name": "sample_data",
"display_name": "sample data",
"description": "sample description",
"data_plane_id": "123e4567-e89b-12d3-a456-426614174000",
"external_id": "ext-dataset-001",
"is_narrative_managed": false,
"schema": {
"file_config": {
"type": "flat",
"delimiter": ",",
"escape": "\"",
"header": true,
"quote": "\""
},
"type": "object",
"properties": {
"idValue": {
"approximate_cardinality": 100000,
"description": "ID values",
"type": "string"
},
"idType": {
"approximate_cardinality": 1,
"type": "string",
"enum": [
"md5_email",
"sha256_email"
]
},
"eventTimestamp": {
"description": "When the observation was collected",
"type": "timestamptz",
"validations": [
"eventTimestamp < current_timestamp()"
]
},
"countryCode": {
"approximate_cardinality": 6,
"description": "The country of residence of the individual",
"type": "string"
},
"age": {
"approximate_cardinality": 90,
"description": "How old this individual is in years",
"type": "long"
},
"birthInfo": {
"description": "The year this individual was born",
"type": "double"
}
},
"required": [
"idValue",
"idType",
"eventTimestamp",
"countryCode",
"age"
],
"primary": "/age"
},
"tags": [
"age",
"demo",
"demographic",
"demographics"
],
"write_mode": "append",
"derive_metrics_config": true
}
'import requests
url = "https://api-dev.narrative.io/datasets"
payload = {
"name": "sample_data",
"display_name": "sample data",
"description": "sample description",
"data_plane_id": "123e4567-e89b-12d3-a456-426614174000",
"external_id": "ext-dataset-001",
"is_narrative_managed": False,
"schema": {
"file_config": {
"type": "flat",
"delimiter": ",",
"escape": "\"",
"header": True,
"quote": "\""
},
"type": "object",
"properties": {
"idValue": {
"approximate_cardinality": 100000,
"description": "ID values",
"type": "string"
},
"idType": {
"approximate_cardinality": 1,
"type": "string",
"enum": ["md5_email", "sha256_email"]
},
"eventTimestamp": {
"description": "When the observation was collected",
"type": "timestamptz",
"validations": ["eventTimestamp < current_timestamp()"]
},
"countryCode": {
"approximate_cardinality": 6,
"description": "The country of residence of the individual",
"type": "string"
},
"age": {
"approximate_cardinality": 90,
"description": "How old this individual is in years",
"type": "long"
},
"birthInfo": {
"description": "The year this individual was born",
"type": "double"
}
},
"required": ["idValue", "idType", "eventTimestamp", "countryCode", "age"],
"primary": "/age"
},
"tags": ["age", "demo", "demographic", "demographics"],
"write_mode": "append",
"derive_metrics_config": True
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
name: 'sample_data',
display_name: 'sample data',
description: 'sample description',
data_plane_id: '123e4567-e89b-12d3-a456-426614174000',
external_id: 'ext-dataset-001',
is_narrative_managed: false,
schema: {
file_config: {type: 'flat', delimiter: ',', escape: '"', header: true, quote: '"'},
type: 'object',
properties: {
idValue: {approximate_cardinality: 100000, description: 'ID values', type: 'string'},
idType: {
approximate_cardinality: 1,
type: 'string',
enum: ['md5_email', 'sha256_email']
},
eventTimestamp: {
description: 'When the observation was collected',
type: 'timestamptz',
validations: ['eventTimestamp < current_timestamp()']
},
countryCode: {
approximate_cardinality: 6,
description: 'The country of residence of the individual',
type: 'string'
},
age: {
approximate_cardinality: 90,
description: 'How old this individual is in years',
type: 'long'
},
birthInfo: {description: 'The year this individual was born', type: 'double'}
},
required: ['idValue', 'idType', 'eventTimestamp', 'countryCode', 'age'],
primary: '/age'
},
tags: ['age', 'demo', 'demographic', 'demographics'],
write_mode: 'append',
derive_metrics_config: true
})
};
fetch('https://api-dev.narrative.io/datasets', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api-dev.narrative.io/datasets",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'name' => 'sample_data',
'display_name' => 'sample data',
'description' => 'sample description',
'data_plane_id' => '123e4567-e89b-12d3-a456-426614174000',
'external_id' => 'ext-dataset-001',
'is_narrative_managed' => false,
'schema' => [
'file_config' => [
'type' => 'flat',
'delimiter' => ',',
'escape' => '"',
'header' => true,
'quote' => '"'
],
'type' => 'object',
'properties' => [
'idValue' => [
'approximate_cardinality' => 100000,
'description' => 'ID values',
'type' => 'string'
],
'idType' => [
'approximate_cardinality' => 1,
'type' => 'string',
'enum' => [
'md5_email',
'sha256_email'
]
],
'eventTimestamp' => [
'description' => 'When the observation was collected',
'type' => 'timestamptz',
'validations' => [
'eventTimestamp < current_timestamp()'
]
],
'countryCode' => [
'approximate_cardinality' => 6,
'description' => 'The country of residence of the individual',
'type' => 'string'
],
'age' => [
'approximate_cardinality' => 90,
'description' => 'How old this individual is in years',
'type' => 'long'
],
'birthInfo' => [
'description' => 'The year this individual was born',
'type' => 'double'
]
],
'required' => [
'idValue',
'idType',
'eventTimestamp',
'countryCode',
'age'
],
'primary' => '/age'
],
'tags' => [
'age',
'demo',
'demographic',
'demographics'
],
'write_mode' => 'append',
'derive_metrics_config' => true
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api-dev.narrative.io/datasets"
payload := strings.NewReader("{\n \"name\": \"sample_data\",\n \"display_name\": \"sample data\",\n \"description\": \"sample description\",\n \"data_plane_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"external_id\": \"ext-dataset-001\",\n \"is_narrative_managed\": false,\n \"schema\": {\n \"file_config\": {\n \"type\": \"flat\",\n \"delimiter\": \",\",\n \"escape\": \"\\\"\",\n \"header\": true,\n \"quote\": \"\\\"\"\n },\n \"type\": \"object\",\n \"properties\": {\n \"idValue\": {\n \"approximate_cardinality\": 100000,\n \"description\": \"ID values\",\n \"type\": \"string\"\n },\n \"idType\": {\n \"approximate_cardinality\": 1,\n \"type\": \"string\",\n \"enum\": [\n \"md5_email\",\n \"sha256_email\"\n ]\n },\n \"eventTimestamp\": {\n \"description\": \"When the observation was collected\",\n \"type\": \"timestamptz\",\n \"validations\": [\n \"eventTimestamp < current_timestamp()\"\n ]\n },\n \"countryCode\": {\n \"approximate_cardinality\": 6,\n \"description\": \"The country of residence of the individual\",\n \"type\": \"string\"\n },\n \"age\": {\n \"approximate_cardinality\": 90,\n \"description\": \"How old this individual is in years\",\n \"type\": \"long\"\n },\n \"birthInfo\": {\n \"description\": \"The year this individual was born\",\n \"type\": \"double\"\n }\n },\n \"required\": [\n \"idValue\",\n \"idType\",\n \"eventTimestamp\",\n \"countryCode\",\n \"age\"\n ],\n \"primary\": \"/age\"\n },\n \"tags\": [\n \"age\",\n \"demo\",\n \"demographic\",\n \"demographics\"\n ],\n \"write_mode\": \"append\",\n \"derive_metrics_config\": true\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api-dev.narrative.io/datasets")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"name\": \"sample_data\",\n \"display_name\": \"sample data\",\n \"description\": \"sample description\",\n \"data_plane_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"external_id\": \"ext-dataset-001\",\n \"is_narrative_managed\": false,\n \"schema\": {\n \"file_config\": {\n \"type\": \"flat\",\n \"delimiter\": \",\",\n \"escape\": \"\\\"\",\n \"header\": true,\n \"quote\": \"\\\"\"\n },\n \"type\": \"object\",\n \"properties\": {\n \"idValue\": {\n \"approximate_cardinality\": 100000,\n \"description\": \"ID values\",\n \"type\": \"string\"\n },\n \"idType\": {\n \"approximate_cardinality\": 1,\n \"type\": \"string\",\n \"enum\": [\n \"md5_email\",\n \"sha256_email\"\n ]\n },\n \"eventTimestamp\": {\n \"description\": \"When the observation was collected\",\n \"type\": \"timestamptz\",\n \"validations\": [\n \"eventTimestamp < current_timestamp()\"\n ]\n },\n \"countryCode\": {\n \"approximate_cardinality\": 6,\n \"description\": \"The country of residence of the individual\",\n \"type\": \"string\"\n },\n \"age\": {\n \"approximate_cardinality\": 90,\n \"description\": \"How old this individual is in years\",\n \"type\": \"long\"\n },\n \"birthInfo\": {\n \"description\": \"The year this individual was born\",\n \"type\": \"double\"\n }\n },\n \"required\": [\n \"idValue\",\n \"idType\",\n \"eventTimestamp\",\n \"countryCode\",\n \"age\"\n ],\n \"primary\": \"/age\"\n },\n \"tags\": [\n \"age\",\n \"demo\",\n \"demographic\",\n \"demographics\"\n ],\n \"write_mode\": \"append\",\n \"derive_metrics_config\": true\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api-dev.narrative.io/datasets")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"name\": \"sample_data\",\n \"display_name\": \"sample data\",\n \"description\": \"sample description\",\n \"data_plane_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"external_id\": \"ext-dataset-001\",\n \"is_narrative_managed\": false,\n \"schema\": {\n \"file_config\": {\n \"type\": \"flat\",\n \"delimiter\": \",\",\n \"escape\": \"\\\"\",\n \"header\": true,\n \"quote\": \"\\\"\"\n },\n \"type\": \"object\",\n \"properties\": {\n \"idValue\": {\n \"approximate_cardinality\": 100000,\n \"description\": \"ID values\",\n \"type\": \"string\"\n },\n \"idType\": {\n \"approximate_cardinality\": 1,\n \"type\": \"string\",\n \"enum\": [\n \"md5_email\",\n \"sha256_email\"\n ]\n },\n \"eventTimestamp\": {\n \"description\": \"When the observation was collected\",\n \"type\": \"timestamptz\",\n \"validations\": [\n \"eventTimestamp < current_timestamp()\"\n ]\n },\n \"countryCode\": {\n \"approximate_cardinality\": 6,\n \"description\": \"The country of residence of the individual\",\n \"type\": \"string\"\n },\n \"age\": {\n \"approximate_cardinality\": 90,\n \"description\": \"How old this individual is in years\",\n \"type\": \"long\"\n },\n \"birthInfo\": {\n \"description\": \"The year this individual was born\",\n \"type\": \"double\"\n }\n },\n \"required\": [\n \"idValue\",\n \"idType\",\n \"eventTimestamp\",\n \"countryCode\",\n \"age\"\n ],\n \"primary\": \"/age\"\n },\n \"tags\": [\n \"age\",\n \"demo\",\n \"demographic\",\n \"demographics\"\n ],\n \"write_mode\": \"append\",\n \"derive_metrics_config\": true\n}"
response = http.request(request)
puts response.read_body{
"id": 18923,
"company_id": 382,
"created_at": "2021-06-24T00:54:40.029056Z",
"description": "Mobile identifiers tied to birth information provided as part of app registration.",
"name": "declared_age",
"display_name": "Declared Age",
"schema": {
"file_config": {
"type": "flat",
"delimiter": ",",
"escape": "\"",
"header": true,
"quote": "\""
},
"type": "object",
"properties": {
"idValue": {
"display_name": "id value",
"order": 0,
"approximate_cardinality": 100000,
"description": "ID values",
"type": "string"
},
"idType": {
"display_name": "id type",
"order": 1,
"approximate_cardinality": 1,
"type": "string",
"enum": [
"md5_email",
"sha256_email"
]
},
"eventTimestamp": {
"display_name": "event timestamp",
"order": 2,
"description": "When the observation was collected",
"type": "timestamptz",
"validations": [
"eventTimestamp < current_timestamp()"
]
},
"countryCode": {
"display_name": "country code",
"order": 3,
"approximate_cardinality": 6,
"description": "The country of residence of the individual",
"type": "string"
},
"age": {
"display_name": "age",
"order": 4,
"approximate_cardinality": 90,
"description": "How old this individual is in years",
"type": "long"
},
"birthInfo": {
"display_name": "birth info",
"order": 5,
"description": "The year this individual was born",
"type": "double"
}
},
"required": [
"idValue",
"idType",
"eventTimestamp",
"countryCode",
"age"
],
"primary": "/age"
},
"status": "active",
"tags": [
"age",
"demo",
"demographic",
"demographics"
],
"updated_at": "2021-06-24T00:54:40.029056Z",
"write_mode": "append",
"subscription_id": "123e4567-e89b-12d3-a456-426614174000",
"last_snapshot_created_at": "2023-06-09T16:51:34.215Z",
"stats": {
"est_total_dataset_stored_files": 10,
"est_total_dataset_stored_records": 500059,
"est_total_dataset_stored_bytes": 4206454,
"active_dataset_stored_files": 10,
"active_dataset_stored_records": 500059,
"active_dataset_stored_bytes": 4206454,
"last_snapshot_added_files": 2,
"last_snapshot_added_records": 499999,
"last_snapshot_deleted_records": 0,
"last_snapshot_added_bytes": 4193500,
"last_snapshot_deleted_files": 0,
"last_snapshot_added_delete_files": 0,
"last_snapshot_removed_delete_files": 0,
"last_snapshot_removed_bytes": 0
}
}{
"error": "Unauthorized",
"error_description": "You are not authorized to use this endpoint."
}Authorizations
Bearer authentication header of the form Bearer <token>, where <token> is your auth token.
Body
application/json
The Dataset's name. Should be unique, contain only alpanum chars and underscores. Max length is 256.
Schema describing a dataset's input file type and the kind of data the file contains. Note: when the file type is "flat" then the order of the properties describing the must be given in the order they appear in the input file.
Show child attributes
Show child attributes
The Dataset's name to display at UI.
A description of the data contained with the dataset.
Optional data plane identifier, must be provided when registering dataset in external dataplane.
Optional external identifier, must be excluded when registering dataset in external dataplane
Whether the dataset is managed by Narrative
Tags that describe the dataset.
How Narrative will treat new data uploaded for the dataset.
appendmeans data will be added to the dataset, this is what to choose if you incrementally update your dataset over time.overwritemeans that all existing data will be overwritten with new data.
Available options:
append, overwrite A flag to enable derivation of the config.
Response
OK
A dataset.
The encoder drops nulls and empty collections, so an optional field is absent rather than null,
and tags and mappings are absent rather than [] when there are none. DatasetAdminResponse
composes this schema with allOf but encodes without that step, so on the admin endpoints the
same optional fields arrive as null. Either way they are optional, and both shapes validate.
Unique identifier for the dataset.
The ID of the company owning the dataset.
The data plane the dataset lives on.
- Option 1
- Option 2
Show child attributes
Show child attributes
Whether Narrative manages the dataset rather than the owning company.
ISO-8601 timestamp indicating when the dataset was created
The Dataset's name. Should be unique, contain only alpanum chars and underscores. Max length is 256.
The Dataset's name to display at UI.
A dataset's retention policies, plus the schedule they are enforced on. This is the shape every dataset response carries; legacy snapshot-only payloads are accepted on write and returned in this form.
Show child attributes
Show child attributes
Example:
{
"policies": [
{
"type": {
"kind": "table_ttl",
"clock": { "kind": "created_at" }
},
"enabled": true,
"interval": "P1D"
}
],
"schedule_config": {
"schedule": "@hourly",
"schedule_zone_id": "UTC",
"status": "active"
}
}
Schema describing a dataset's input file type and the kind of data the file contains. Note: when the file type is "flat" then the order of the properties describing the must be given in the order they appear in the input file.
Show child attributes
Show child attributes
The dataset status.
activemeans that data can be added to the dataset but the schema is locked in placependingmeans the dataset has been created, but the schema can still be altered and no data can be added to the datasetarchivedmeans the dataset has been retired and can no longer be read from or written to
Available options:
active, pending, archived How Narrative will treat new data uploaded for the dataset.
appendmeans data will be added to the dataset, this is what to choose if you incrementally update your dataset over time.overwritemeans that all existing data will be overwritten with new data.
Available options:
append, overwrite ISO-8601 timestamp indicating when the dataset was last updated.
The dataset's identifier in the system it came from. Absent when the dataset did not come from one.
The mappings from this dataset's fields to attributes. Absent when the dataset has none.
Show child attributes
Show child attributes
Compute pool configuration for the dataset.
Show child attributes
Show child attributes
A description of the data contained with the dataset.
Tags that describe the dataset.
Id of a subscription which writes output to the dataset
ISO-8601 timestamp indicating when the last snapshot was created.
Materialized-view configuration. Present only for materialized-view datasets.
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Was this page helpful?
⌘I