Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@stefanoverna
stefanoverna committed Dec 23, 2024
1 parent 17ff0b1 commit 150d78f
Showing 1 changed file with 287 additions and 17 deletions.
304 changes: 287 additions & 17 deletions migrations/1734676047_docPages.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { Client, SchemaTypes } from '@datocms/cli/lib/cma-client-node';
import { render, type StructuredTextDocument } from 'datocms-structured-text-to-plain-text';
import { isLink, type Link } from 'datocms-structured-text-utils';
import { isLink, type ItemLink, type NodeWithMeta } from 'datocms-structured-text-utils';
import { cloneDeep, isEqual, omit } from 'lodash-es';
import { match } from 'path-to-regexp';
import { visit } from 'unist-util-visit';

Expand All @@ -17,6 +18,7 @@ const knownPaths = [
'/contact/thanks',
'/customer-stories',
'/customers',
'/changelog',
'/docs/community-tutorials',
'/docs/content-delivery-api/filtering-records',
'/docs/content-delivery-api/filtering-uploads',
Expand Down Expand Up @@ -385,7 +387,7 @@ const pathToResourceConfig = [
field: 'slug',
},
{
pattern: '/marketplace/plugins/i/:packageName',
pattern: '/marketplace/plugins/i/*packageName',
urlVariable: 'packageName',
model: 'plugin',
field: 'package_name',
Expand Down Expand Up @@ -434,19 +436,240 @@ const pathToResourceConfig = [
},
];

const redirects = Object.entries({
'/marketplace/plugins/i/datocms-plugin-yoast-seo':
'/marketplace/plugins/i/datocms-plugin-seo-readability-analysis',
'/docs/introduction/custom-assets-domain': '/marketplace/enterprise/aws-s3',
'/search': '/docs/site-search',
'/docs/search': '/docs/site-search',
'/about': '/company/about',
'/brand-assets': '/company/brand-assets',
'/docs/building-plugins/*splat': '/docs/legacy-plugins/*splat',
'/docs/guides/custom-assets-domain/*splat': '/marketplace/enterprise/*splat',
'/docs/guides/single-sign-on': '/marketplace/enterprise',
'/docs/guides/single-sign-on/configure-sso-with-okta': '/marketplace/enterprise/okta-sso',
'/docs/guides/single-sign-on/configure-sso-with-onelogin': '/marketplace/enterprise/onelogin-sso',
'/docs/deployments/travis': '/marketplace/hosting/travis-ci',
'/docs/deployments/*splat': '/marketplace/hosting/*splat',
'/plugins/i/@stackbitdatocms-plugin-typed-list':
'/marketplace/plugins/i/@stackbit/datocms-plugin-typed-list',
'/plugins/i/@ecologicdatocms-plugin-multiselect':
'/marketplace/plugins/i/@ecologic/datocms-plugin-multiselect',
'/plugins/i/@ecologicdatocms-plugin-ordered-tag-editor':
'/marketplace/plugins/i/@ecologic/datocms-plugin-ordered-tag-editor',
'/content-management-api': '/docs/content-management-api',
'/plugins': '/marketplace/plugins',
'/plugins/i/*splat': '/marketplace/plugins/i/*splat',
'/docs/plugins/entry-point': '/docs/building-plugins/entry-point',
'/docs/plugins/creating-a-new-plugin': '/docs/building-plugins/creating-a-new-plugin',
'/docs/plugins/publishing': '/docs/building-plugins/publishing',
'/docs/guides/offline-backups': '/docs/import-and-export/export-data',
'/docs/general-concepts/transfer': '/docs/plans-pricing-and-billing/transfer',
'/docs/content-management-api/js-client': '/docs/content-management-api/using-the-nodejs-clients',
'/docs/security': '/security',
'/docs/guides/private-videos': '/docs/content-modelling/external-video-field',
'/docs/content-delivery-api/overview': '/docs/content-delivery-api',
'/docs/content-delivery-api/endpoint': '/docs/content-delivery-api/api-endpoints',
'/docs/content-delivery-api/complexity_limiting': '/docs/content-delivery-api/complexity',
'/docs/content-delivery-api/first-request': '/docs/content-delivery-api/your-first-request',
'/docs/content-delivery-api/querying': '/docs/content-delivery-api/how-to-fetch-records',
'/docs/content-delivery-api/filtering': '/docs/content-delivery-api/filtering-records',
'/docs/content-delivery-api/ordering': '/docs/content-delivery-api/ordering-records',
'/docs/content-delivery-api/modular-content': '/docs/content-delivery-api/modular-content-fields',
'/docs/content-delivery-api/uploads': '/docs/content-delivery-api/images-and-videos',
'/docs/content-delivery-api/seo': '/docs/content-delivery-api/seo-and-favicon',
'/blog/2': '/blog/p/2',
'/changelog/2': '/product-updates/p/2',
'/changelog/3': '/product-updates/p/3',
'/changelog/4': '/product-updates/p/4',
'/changelog/5': '/product-updates/p/5',
'/changelog/6': '/product-updates/p/5',
'/changelog/*splat': '/product-updates/*splat',
'/docs/general-concepts/pricing': '/docs/plans-pricing-and-billing',
'/docs/guides/installing-site-search/*splat': '/docs/site-search/*splat',
'/docs/guides/*splat': '/docs/*splat',
'/docs/static-generators/gatsbyjs/*splat': '/docs/gatsby/*splat',
'/docs/static-generators/*splat': '/docs/*splat',
'/docs/single-page-apps/*splat': '/docs/*splat',
'/marketplace/hosting/zeit': '/marketplace/hosting/vercel',
'/cms/react': '/cms/react-cms',
'/cms/hugo': '/cms/hugo-cms',
'/cms/nextjs': '/cms/nextjs-cms',
'/cms/nuxt-js': '/cms/nuxtjs-cms',
'/cms/jekyll': '/cms/jekyll-cms',
'/cms/middleman': '/cms/middleman-cms',
'/cms/vue': '/cms/vue-js-cms',
'/cms/gatsbyjs': '/cms/gatsbyjs-cms',
'/features/video-streaming-encoding': '/features/video-api',
'/features/graphql-content-api': '/features/headless-cms-graphql',
'/features/multi-language': '/features/headless-cms-multi-language',
'/features/workflows': '/features/workflow-cms',
'/features/real-time': '/features/real-time-api',
'/features/structured-text': '/features/structured-content-cms',
'/team/digital-marketers': '/team/cms-digital-marketing',
'/blog/what-is-an-headless-cms': '/blog/what-is-a-headless-cms',
'/team/developers': '/team/best-cms-for-developers',
'/enterprise': '/enterprise-headless-cms',
'/marketplace/starters/nextjs-blog': '/marketplace/starters/nextjs-template-blog',
'/blog/offer-responsive-progressive-lqip-images-in-2020':
'/blog/best-way-for-handling-react-images',
'/blog/static-ecommerce-website-snipcart-gatsbyjs-datocms': '/blog/gatsby-ecommerce-tutorial',
'/cms/gatsbyjs-cms': '/cms/gatsby-cms',
'/legal/security': '/security',
'/legal/privacy': '/legal/privacy-policy',
'/docs/pro-tips/manage-draft-published-state-by-locale':
'/docs/general-concepts/localization#locale-based-publishing',
'/marketplace/starters/next-js-multilingual-blog-per-local-publishing':
'/docs/general-concepts/localization#locale-based-publishing',
'/pricing/compare': '/pricing',
'/docs/content-management-api/rate-limits': '/docs/content-management-api/technical-limits',
'/docs/content-delivery-api/rate-limiting': '/docs/content-delivery-api/technical-limits',
'/partners/locale': '/tech-partners/crowdin',
'/partners/imgix': '/tech-partners/imgix',
'/blog/introducing-visual-editing-for-vercel-and-datocms-enterprise-customers':
'/blog/introducing-content-link-for-vercel-and-datocms-enterprise-customers',
'/blog/july-update-visual-editing-and-per-locale-publishing':
'/blog/july-update-content-link-and-per-locale-publishing',
'/docs/visual-editing': '/docs/content-link/how-to-use-content-link',
'/docs/visual-editing/how-to-use-visual-editing': '/docs/content-link/how-to-use-content-link',
'/partners/harvey-cameron/showcase/jacuzzi ': '/partners',
'/docs/plugins/install': '/docs/general-concepts/plugins',
'/blog/live-preview-changes-on-gatsby-preview': '/blog',
'/docs/localizing-images': '/docs/content-delivery-api/images-and-videos',
'/cda-explorer': '/docs/content-delivery-api',
'/docs/content-management-api/using-the-ruby-client': '/docs/content-management-api',
'/use-cases': '/',
'/docs/cdn-settings/advanced-asset-settings': '/docs/asset-api/asset-cdn-settings',
'/docs/general-concepts/videos': '/docs/asset-api/videos',
'/docs/general-concepts/images': '/docs/asset-api/images',
'/docs/project-starters-and-templates': '/docs/general-concepts/project-starters-and-templates',
'/docs/project-starters-and-templates/clone-project-button':
'/docs/general-concepts/project-starters-and-templates#generate-a-clone-project-button',
'/docs/project-starters-and-templates/project-starter-button':
'/docs/general-concepts/project-starters-and-templates#generate-a-project-starter-button',
'/blog/headless-cms-unconventional-use-cases': '/customer-stories/trip-to-japan',
'/docs/plugin-sdk': '/docs/plugin-sdk/introduction',
'/docs/general-concepts/environments': '/docs/general-concepts/primary-and-sandbox-environments',
'/docs/react/*splat': '/docs/next-js/*splat',
'/marketplace/plugins/i/datocms-plugin-gatsby-cloud': '/',
'/docs/content-management-api/resources/sso-token':
'/docs/content-management-api/resources/sso-settings/generate_token',
'/docs/next-js/setting-up-next-js-preview-mode':
'/docs/legacy-next-js-documentation/setting-up-next-js-preview-mode',
'/docs/plugin-sdk/execute-code-on-boot': '/docs/plugin-sdk/event-hooks',
'/partners/harvey-cameron': '/partners',
'/docs/vue/display-videos': '/docs/nuxt',
'/docs/vue/loading-responsive-progressive-images-from-datocms': '/docs/nuxt',
'/docs/vue': '/docs/nuxt',
'/docs/vue/*splat': '/docs/nuxt/*splat',
'/docs/sveltekit/getting-started-with-sveltekit-and-datocms': '/docs/svelte',
'/partners/dev-kitchen': '/partners',
'/docs/plugin-sdk/sdk/field-extensions ': '/docs/plugin-sdk/field-extensions',
'/cms/hugo-cms': '/',
'/marketplace/starters/next-13-company-landing-page-demo':
'/marketplace/starters/next-js-starter-kit',
'/docs/agency-partner-program/benefits': '/partner-program',
'/docs/site-search/excluding-text':
'/docs/site-search/how-the-crawling-works#excluding-content-from-indexing',
'/docs/building-plugins/creating-a-new-plugin': '/docs/plugin-sdk/introduction',
'/docs/building-plugins/sdk-reference': '/docs/plugin-sdk/introduction',
'/docs/building-plugins/entry-point': '/docs/plugin-sdk/introduction',
'/docs/gatsby': '/docs',
'/docs/other-ssgs': '/docs',
'/docs/guides/building-plugins/sdk-reference': '/docs/plugin-sdk/build-your-first-plugin',
'/docs/react': '/docs',
'/docs/guides/building-plugins/creating-a-new-plugin': '/docs/plugin-sdk/build-your-first-plugin',
'/docs/hugo': '/docs',
'/docs/building-plugins': '/docs/plugin-sdk/build-your-first-plugin',
'/docs/building-plugins/using-the-generator': '/docs/plugin-sdk/build-your-first-plugin',
'/docs/content-delivery-api/docs/general-concepts/roles-and-permission-system':
'/docs/general-concepts/roles-and-permission-system',
'/docs/plugin-sdk/render%E2%80%8BUpload%E2%80%8BSidebarPanel': '/docs/plugin-sdk/sidebar-panels',
'/docs/plugin-sdk/sdk/field-extensions': '/docs/plugin-sdk/field-extensions',
'/docs/building-plugins/install':
'/docs/plugin-sdk/build-your-first-plugin#install-your-plugin-in-the-datocms-web-app',
'/docs/building-plugins/publishing': '/docs/plugin-sdk/publishing-to-marketplace',
'/docs/guides': '/docs',
'/docs/react/managing-images': '/docs',
'/marketplace/plugins/i/[email protected]:voorhoede/datocms-plugin-editor-help.git':
'/marketplace/plugins',
'/marketplace/plugins/i/license': '/marketplace/plugins',
'/docs/gatsby/gatsby-cloud': '/docs',
'/docs/single-page-apps/react': '/docs',
'/docs/static-generators/gatsbyjs': '/docs',
'/marketplace/plugins/i/contributing.md': '/docs/plugin-sdk/publishing-to-marketplace',
'/marketplace/starters/middleman-portfolio': '/marketplace/starters',
'/docs/react/structured-text-fields': '/docs',
'/marketplace/plugins/i/LICENSE.txt': '/docs/plugin-sdk/publishing-to-marketplace',
'/docs/middleman': '/docs',
'/docs/guides/localizing-images': '/docs/asset-api/images',
'/blog/introducing-inherited-roles-for-greater-modularity-in-role-management':
'/blog/introducing-inherited-roles',
'/marketplace/plugins/i/datocms-plugin-kontainer': '/marketplace/plugins',
'/docs/vue/rendering-structured-text-fields': '/docs/nuxt/rendering-structured-text-fields',
'/product-updates/undefined': '/product-updates',
'/docs/content-management-api/resources/item/instances/docs/content-management-api/resources/item/instances':
'/docs/content-management-api',
'/marketplace/starters/next-js-multilingual-blog': '/marketplace/starters',
'/docs/guides/installing-site-search/widget': '/docs/general-concepts/site-search',
'/docs/pro-tips/create-a-select-with-a-single-line-string-field': '/docs',
'/docs/guides/building-plugins': '/docs/plugin-sdk/build-your-first-plugin',
'/partners/tecnica': '/partners',
'/docs/hugo/fields': '/docs',
'/docs/content-modelling/data-migration/docs/content-modelling/data-migration':
'/docs/content-modelling/data-migration',
'/partners/harvey-cameron/showcase/jacuzzi': '/partners',
'/docs/plugin-sdk/field-extension': '/docs/plugin-sdk/field-extensions',
'/docs/jekyll/fields': '/docs',
'/docs/static-generators/middleman/image-manipulation': '/docs',
'/docs/gatsby/examples': '/docs',
'/docs/guides/custom-assets-domain/google-cloud-storage':
'/marketplace/enterprise/google-cloud-storage',
'/use-cases/docs': '/use-cases/headless-cms-knowledge-management',
'/features/headless-cms-multi-languag': '/features/headless-cms-multi-language',
'/docs/guides/custom-assets-domain': '/docs',
'/docs/deployments/zeit': '/docs',
'/docs/content-management-api/resources/upload/updatebut':
'/docs/content-management-api/resources/upload',
'/foo': '/docs',
}).map(([pattern, replacement]) => ({
matcher: match(pattern, { decode: decodeURIComponent }),
replacer: (params: Partial<Record<string, string | string[]>>) =>
replacement.replace('*splat', params?.splat ? (params.splat as string[]).join('/') : '*SPLAT'),
}));

async function fetchItemIdForPathname(
client: Client,
pathname: string,
): Promise<[string, string] | 'ITEM_NOT_FOUND' | 'INVALID_ROUTE'> {
): Promise<[string, string] | 'ITEM_NOT_FOUND' | 'INVALID_ROUTE' | 'MULTIPLE_ITEMS'> {
for (const config of pathToResourceConfig) {
const matcher = match(config.pattern, { decode: decodeURIComponent });
const result = matcher(pathname);

if (result) {
const fieldValue = result.params[config.urlVariable];
const items = await client.items.list({
filter: { type: config.model, fields: { [config.field]: { eq: fieldValue } } },
let { data: items } = await client.items.rawList({
filter: {
type: config.model,
fields: {
[config.field]: { eq: Array.isArray(fieldValue) ? fieldValue.join('/') : fieldValue },
},
},
});
if (config.model === 'doc_page') {
const docGroupSlug = result.params.docGroupSlug as string;
const newItems: SchemaTypes.Item[] = [];
for (const item of items) {
const docGroups = await client.items.references(item, { version: 'published' });
if (docGroups.find((group) => group.slug === docGroupSlug)) {
newItems.push(item);
}
}
items = newItems;
}
if (items.length > 1) {
return 'MULTIPLE_ITEMS';
}
const firstItem = items[0];
if (!firstItem) {
return 'ITEM_NOT_FOUND';
Expand All @@ -470,7 +693,7 @@ export default async function (client: Client) {

async function analyzeItem(
item: SchemaTypes.Item,
cb: (path: string, url: URL, node: Link) => Promise<void>,
cb: (path: string, url: URL, node: NodeWithMeta) => Promise<void>,
path: string[] = [],
) {
const itemType = itemTypes.find((it) => it.id === item.relationships.item_type.data.id)!;
Expand Down Expand Up @@ -499,6 +722,20 @@ export default async function (client: Client) {
try {
const url = new URL(node.url, 'https://www.datocms.com/');
if (url.hostname === 'www.datocms.com') {
if (url.pathname.endsWith('/')) {
// console.log('TRAILING_SLASH', url.pathname);
url.pathname = url.pathname.slice(0, -1);
}

for (const redirect of redirects) {
const result = redirect.matcher(url.pathname);
if (result) {
const newPathname = redirect.replacer(result.params);
// console.log('REDIRECT', url.pathname, '->', newPathname);
url.pathname = newPathname;
}
}

promises.push(cb(path.join('.'), url, node));
}
} catch {
Expand Down Expand Up @@ -537,32 +774,65 @@ export default async function (client: Client) {
filter: { type: model.id },
nested: true,
})) {
await analyzeItem(record, async (path, url, link) => {
const oldRecord = cloneDeep(record);

await analyzeItem(record, async (path, url, node) => {
const result = await fetchItemIdForPathname(client, url.pathname);
if (Array.isArray(result)) {
const [modelApiKey, itemId] = result;
const linkedModel = itemTypes.find((it) => it.attributes.api_key === modelApiKey)!;

const itemLinkNode = node as ItemLink;
itemLinkNode.type = 'itemLink';
itemLinkNode.item = itemId;

const meta = itemLinkNode.meta || [];
if (url.hash) {
itemLinkNode.meta?.push({ id: 'hash', value: url.hash });
}
if (url.search) {
itemLinkNode.meta?.push({ id: 'search', value: url.search });
}
if (meta.length > 0) {
itemLinkNode.meta = meta;
}
delete (itemLinkNode as any).url;

console.log(
'[FOUND]',
`https://datocms.admin.datocms.com/editor/item_types/${model.id}/items/${record.id}`,
path,
url.toString(),
`"${render(link)}"`,
`"${render(node)}"`,
`https://datocms.admin.datocms.com/editor/item_types/${linkedModel.id}/items/${itemId}`,
JSON.stringify(itemLinkNode),
);
} else if (
(result === 'INVALID_ROUTE' && !knownPaths.includes(url.pathname)) ||
result === 'ITEM_NOT_FOUND'
!knownPaths.includes(url.pathname) &&
(result === 'INVALID_ROUTE' || result === 'ITEM_NOT_FOUND')
) {
console.log(
`[${result}]`,
`https://datocms.admin.datocms.com/editor/item_types/${model.id}/items/${record.id}`,
path,
url.toString(),
`"${render(link)}"`,
);
// console.log(
// `[${result}]`,
// `https://datocms.admin.datocms.com/editor/item_types/${model.id}/items/${record.id}`,
// path,
// url.toString(),
// `"${render(node)}"`,
// );
} else if (result === 'MULTIPLE_ITEMS') {
// console.log(
// `[${result}]`,
// `https://datocms.admin.datocms.com/editor/item_types/${model.id}/items/${record.id}`,
// path,
// url.toString(),
// `"${render(node)}"`,
// );
}
});

if (!isEqual(oldRecord, record)) {
await client.items.rawUpdate(record.id, { data: omit(record, 'relationships') });
console.log('Updated');
}
}
}

Expand Down

0 comments on commit 150d78f

Please sign in to comment.