Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Contexts -> Breakouts, Metrics -> Aggregations #28

Merged
merged 3 commits into from
Feb 18, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions lib/manifold/api/schema_generator.rb
Original file line number Diff line number Diff line change
Expand Up @@ -33,39 +33,39 @@ def manifold_schema
private

def metrics_fields
return [] unless @manifold_yaml["contexts"] && @manifold_yaml["metrics"]
return [] unless @manifold_yaml["breakouts"] && @manifold_yaml["aggregations"]

@manifold_yaml["contexts"].map do |context_name, _context_config|
@manifold_yaml["breakouts"].map do |breakout_name, _breakout_config|
{
"name" => context_name,
"name" => breakout_name,
"type" => "RECORD",
"mode" => "NULLABLE",
"fields" => context_metrics_fields
"fields" => breakout_metrics_fields
}
end
end

def context_metrics_fields
def breakout_metrics_fields
[
*countif_fields,
*sumif_fields
]
end

def countif_fields
return [] unless @manifold_yaml.dig("metrics", "countif")
return [] unless @manifold_yaml.dig("aggregations", "countif")

[{
"name" => @manifold_yaml["metrics"]["countif"],
"name" => @manifold_yaml["aggregations"]["countif"],
"type" => "INTEGER",
"mode" => "NULLABLE"
}]
end

def sumif_fields
return [] unless @manifold_yaml.dig("metrics", "sumif")
return [] unless @manifold_yaml.dig("aggregations", "sumif")

@manifold_yaml["metrics"]["sumif"].keys.map do |metric_name|
@manifold_yaml["aggregations"]["sumif"].keys.map do |metric_name|
{
"name" => metric_name,
"type" => "INTEGER",
Expand Down
2 changes: 1 addition & 1 deletion lib/manifold/api/workspace.rb
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ def generate_dimensions_merge_sql(source_sql)
def valid_manifold_config?
return false unless @manifold_yaml

%w[source timestamp.field contexts metrics].all? do |field|
%w[source timestamp.field breakouts aggregations].all? do |field|
@manifold_yaml&.dig(*field.split("."))
end
end
Expand Down
4 changes: 2 additions & 2 deletions lib/manifold/templates/workspace_template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ timestamp:
interval: HOUR
field: timestamp

contexts:
breakouts:
paid: IS_PAID(context.location)
organic: IS_ORGANIC(context.location)
paidOrganic:
Expand All @@ -21,7 +21,7 @@ contexts:
- organic
operator: AND

metrics:
aggregations:
countif: tapCount
sumif:
sequenceSum:
Expand Down
24 changes: 12 additions & 12 deletions lib/manifold/terraform/workspace_configuration.rb
Original file line number Diff line number Diff line change
Expand Up @@ -9,39 +9,39 @@ def initialize(manifold_config)
end

def build_metrics_struct
return "" unless @manifold_config&.dig("contexts") && @manifold_config&.dig("metrics")
return "" unless @manifold_config&.dig("breakouts") && @manifold_config&.dig("aggregations")

context_structs = @manifold_config["contexts"].map do |name, config|
condition = build_context_condition(name, config)
metrics = build_context_metrics(condition)
breakout_structs = @manifold_config["breakouts"].map do |name, config|
condition = build_breakout_condition(name, config)
metrics = build_breakout_metrics(condition)
"\tSTRUCT(\n\t\t#{metrics}\n\t) AS #{name}"
end

context_structs.join(",\n")
breakout_structs.join(",\n")
end

private

def build_context_metrics(condition)
def build_breakout_metrics(condition)
metrics = []
add_count_metrics(metrics, condition)
add_sum_metrics(metrics, condition)
metrics.join(",\n\t\t")
end

def add_count_metrics(metrics, condition)
return unless @manifold_config.dig("metrics", "countif")
return unless @manifold_config.dig("aggregations", "countif")

metrics << "COUNTIF(#{condition}) AS #{@manifold_config["metrics"]["countif"]}"
metrics << "COUNTIF(#{condition}) AS #{@manifold_config["aggregations"]["countif"]}"
end

def add_sum_metrics(metrics, condition)
@manifold_config.dig("metrics", "sumif")&.each do |name, config|
@manifold_config.dig("aggregations", "sumif")&.each do |name, config|
metrics << "SUM(IF(#{condition}, #{config["field"]}, 0)) AS #{name}"
end
end

def build_context_condition(_name, config)
def build_breakout_condition(_name, config)
return config unless config.is_a?(Hash)

operator = config["operator"]
Expand All @@ -50,7 +50,7 @@ def build_context_condition(_name, config)
end

def build_operator_condition(operator, fields)
conditions = fields.map { |f| @manifold_config["contexts"][f] }
conditions = fields.map { |f| @manifold_config["breakouts"][f] }
case operator
when "AND", "OR" then join_conditions(conditions, operator)
when "NOT" then negate_condition(conditions.first)
Expand Down Expand Up @@ -325,7 +325,7 @@ def valid_manifold_config?
end

def required_fields_present?
%w[source timestamp.field contexts metrics].all? do |field|
%w[source timestamp.field breakouts aggregations].all? do |field|
@manifold_config&.dig(*field.split("."))
end
end
Expand Down
40 changes: 20 additions & 20 deletions spec/manifold/api/workspace_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@
workspace.manifold_path.write(<<~YAML)
vectors:
- User
contexts:
breakouts:
paid: IS_PAID(context.location)
organic: IS_ORGANIC(context.location)
paidOrganic:
Expand Down Expand Up @@ -117,7 +117,7 @@
- paid
- organic
operator: XNOR
metrics:
aggregations:
countif: tapCount
sumif:
sequenceSum:
Expand Down Expand Up @@ -173,38 +173,38 @@
expect(schema_fields[:metrics]["mode"]).to eq("REQUIRED")
end

shared_examples "context metrics" do |context_name|
let(:context) { schema_fields[:metrics]["fields"].find { |f| f["name"] == context_name } }
shared_examples "breakout metrics" do |breakout_name|
let(:breakout) { schema_fields[:metrics]["fields"].find { |f| f["name"] == breakout_name } }

it "includes tapCount metric" do
expect(context["fields"]).to include(
expect(breakout["fields"]).to include(
{ "type" => "INTEGER", "name" => "tapCount", "mode" => "NULLABLE" }
)
end

it "includes sequenceSum metric" do
expect(context["fields"]).to include(
expect(breakout["fields"]).to include(
{ "type" => "INTEGER", "name" => "sequenceSum", "mode" => "NULLABLE" }
)
end
end

include_examples "context metrics", "paid"
include_examples "context metrics", "organic"
include_examples "context metrics", "paidOrganic"
include_examples "context metrics", "paidOrOrganic"
include_examples "context metrics", "notPaid"
include_examples "context metrics", "neitherPaidNorOrganic"
include_examples "context metrics", "notBothPaidAndOrganic"
include_examples "context metrics", "eitherPaidOrOrganic"
include_examples "context metrics", "similarPaidOrganic"
include_examples "breakout metrics", "paid"
include_examples "breakout metrics", "organic"
include_examples "breakout metrics", "paidOrganic"
include_examples "breakout metrics", "paidOrOrganic"
include_examples "breakout metrics", "notPaid"
include_examples "breakout metrics", "neitherPaidNorOrganic"
include_examples "breakout metrics", "notBothPaidAndOrganic"
include_examples "breakout metrics", "eitherPaidOrOrganic"
include_examples "breakout metrics", "similarPaidOrganic"

it "includes all contexts in the metrics fields" do
it "includes all breakouts in the metrics fields" do
expect(schema_fields[:metrics]["fields"].map { |f| f["name"] })
.to match_array(expected_context_names)
.to match_array(expected_breakout_names)
end

def expected_context_names
def expected_breakout_names
%w[
paid organic paidOrganic paidOrOrganic notPaid
neitherPaidNorOrganic notBothPaidAndOrganic
Expand Down Expand Up @@ -329,9 +329,9 @@ def manifold_yaml_content
timestamp:
field: created_at
interval: DAY
contexts:
breakouts:
paid: IS_PAID(context.location)
metrics:
aggregations:
countif: tapCount
YAML
end
Expand Down
16 changes: 8 additions & 8 deletions spec/manifold/terraform/metrics_builder_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

let(:manifold_config) do
{
"contexts" => {
"breakouts" => {
"paid" => "IS_PAID(context.location)",
"organic" => "IS_ORGANIC(context.location)",
"paidOrganic" => {
Expand Down Expand Up @@ -37,7 +37,7 @@
"operator" => "XNOR"
}
},
"metrics" => {
"aggregations" => {
"countif" => "tapCount",
"sumif" => {
"sequenceSum" => {
Expand All @@ -53,13 +53,13 @@

context "with valid configuration" do
it "wraps each context in STRUCT" do
manifold_config["contexts"].each_key do |_context|
manifold_config["breakouts"].each_key do |_context|
expect(metrics_struct).to include("STRUCT(")
end
end

it "includes each context name" do
manifold_config["contexts"].each_key do |context|
manifold_config["breakouts"].each_key do |context|
expect(metrics_struct).to include(") AS #{context}")
end
end
Expand All @@ -81,14 +81,14 @@
end
end

context "when no contexts are defined" do
let(:manifold_config) { { "metrics" => {} } }
context "when no breakouts are defined" do
let(:manifold_config) { { "breakouts" => {} } }

it { is_expected.to eq("") }
end

context "when no metrics are defined" do
let(:manifold_config) { { "contexts" => {} } }
context "when no aggregations are defined" do
let(:manifold_config) { { "aggregations" => {} } }

it { is_expected.to eq("") }
end
Expand Down
12 changes: 6 additions & 6 deletions spec/manifold/terraform/workspace_configuration_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@
let(:name) { "analytics" }
let(:manifold_config) do
{
"contexts" => {
"breakouts" => {
"paid" => "IS_PAID(context.location)"
},
"metrics" => {
"aggregations" => {
"countif" => "tapCount"
},
"source" => "analytics.events",
Expand Down Expand Up @@ -112,10 +112,10 @@
timestamp:
field: #{manifold_config["timestamp"]["field"]}
interval: #{manifold_config["timestamp"]["interval"]}
contexts:
paid: #{manifold_config["contexts"]["paid"]}
metrics:
countif: #{manifold_config["metrics"]["countif"]}
breakouts:
paid: #{manifold_config["breakouts"]["paid"]}
aggregations:
countif: #{manifold_config["aggregations"]["countif"]}
filter: #{manifold_config["filter"]}
YAML
workspace.write_manifold_merge_sql
Expand Down
4 changes: 2 additions & 2 deletions spec/support/shared_contexts/with_template_files.rb
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@
timestamp:
field: created_at
interval: DAY
contexts:
breakouts:
paid: IS_PAID(context.location)
metrics:
aggregations:
countif: tapCount
YAML
vector_template_path.write("attributes:")
Expand Down