Skip to content

Commit b0c0e09

Browse files
mdawaffedereksmart
mdawaffe
and
dereksmart
authored
Blog Privacy [jetpack-mu-wpcom]: Add AI User Agents to robots.txt depending on blog setting (#35704)
* Blog Privacy [jetpack-mu-wpcom]: Add AI User Agents to robots.txt * changelog * Alphabetize, remove BingBot, add GPTBot * Discourage AI bots when blog_public==0 * blog_public===0 already discourages all bots. Typo * Adjust filter priority * tools/project-version.sh -f -u 5.13.0-alpha packages/jetpack-mu-wpcom --------- Co-authored-by: dereksmart <[email protected]>
1 parent 7bb8b6b commit b0c0e09

File tree

11 files changed

+199
-8
lines changed

11 files changed

+199
-8
lines changed

‎projects/packages/jetpack-mu-wpcom/changelog/add-mu-wpcom-ai-robots-txt-blocks

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Significance: minor
2+
Type: added
3+
4+
Blog Privacy: Add AI User Agents to robots.txt depending on blog setting.

‎projects/packages/jetpack-mu-wpcom/composer.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@
5050
},
5151
"autotagger": true,
5252
"branch-alias": {
53-
"dev-trunk": "5.12.x-dev"
53+
"dev-trunk": "5.13.x-dev"
5454
},
5555
"textdomain": "jetpack-mu-wpcom",
5656
"version-constants": {

‎projects/packages/jetpack-mu-wpcom/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"private": true,
33
"name": "@automattic/jetpack-mu-wpcom",
4-
"version": "5.12.3-alpha",
4+
"version": "5.13.0-alpha",
55
"description": "Enhances your site with features powered by WordPress.com",
66
"homepage": "https://github.com/Automattic/jetpack/tree/HEAD/projects/packages/jetpack-mu-wpcom/#readme",
77
"bugs": {

‎projects/packages/jetpack-mu-wpcom/src/class-jetpack-mu-wpcom.php

+3-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
* Jetpack_Mu_Wpcom main class.
1414
*/
1515
class Jetpack_Mu_Wpcom {
16-
const PACKAGE_VERSION = '5.12.3-alpha';
16+
const PACKAGE_VERSION = '5.13.0-alpha';
1717
const PKG_DIR = __DIR__ . '/../';
1818
const BASE_DIR = __DIR__ . '/';
1919
const BASE_FILE = __FILE__;
@@ -78,6 +78,8 @@ public static function load_features() {
7878
require_once __DIR__ . '/features/block-patterns/block-patterns.php';
7979

8080
require_once __DIR__ . '/features/wpcom-site-menu/wpcom-site-menu.php';
81+
82+
require_once __DIR__ . '/features/blog-privacy/blog-privacy.php';
8183
}
8284

8385
/**

‎projects/packages/jetpack-mu-wpcom/src/features/blog-privacy/blog-privacy.php

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
<?php
2+
/**
3+
* Blog Privacy Settings
4+
*
5+
* Controls for Blog Privacy are spread out over several different places.
6+
* * WordPress Core (all sites: `blog_privacy_selector`, `blog_public`)
7+
* * wpcomsh (WoA: Private Site feature)
8+
* * WP.com (simple: `blog_public`)
9+
* * This jetpack-mu-wpcom feature (WoA, simple: `wpcom_data_sharing_opt_out` robots.txt user agent blocks)
10+
*
11+
* @package automattic/jetpack-mu-wpcom
12+
*/
13+
14+
declare( strict_types = 1 );
15+
16+
namespace Automattic\Jetpack\Jetpack_Mu_Wpcom\Blog_Privacy;
17+
18+
/**
19+
* Filters the robots.txt contents based on the value of the wpcom_data_sharing_opt_out option.
20+
*
21+
* @param string $output The contents of robots.txt.
22+
* @param int|string $public The value of the blog_public option.
23+
* @return string
24+
*/
25+
function robots_txt( string $output, $public ): string {
26+
$public = (int) $public;
27+
28+
// If the site is completely private or already discouraging *all* bots, don't bother with the additional restrictions.
29+
if ( -1 === $public || 0 === $public ) {
30+
return $output;
31+
}
32+
33+
// An option oddly named because of history.
34+
if ( get_option( 'wpcom_data_sharing_opt_out' ) ) {
35+
$ai_bots = array(
36+
'Amazonbot',
37+
'CCBot',
38+
'FacebookBot',
39+
'Google-Extended',
40+
'GPTBot',
41+
'omgili',
42+
'omgilibot',
43+
'SentiBot',
44+
'sentibot',
45+
);
46+
47+
foreach ( $ai_bots as $ai_bot ) {
48+
$output .= "\nUser-agent: {$ai_bot}\n";
49+
$output .= "Disallow: /\n";
50+
}
51+
}
52+
53+
return $output;
54+
}
55+
56+
add_filter( 'robots_txt', __NAMESPACE__ . '\\robots_txt', 12, 2 );

‎projects/packages/jetpack-mu-wpcom/tests/php/features/blog-privacy/class-blog-privacy-test.php

+124
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
<?php
2+
/**
3+
* Blog Privacy Tests
4+
*
5+
* @package automattic/jetpack-mu-wpcom
6+
*/
7+
8+
declare( strict_types = 1 );
9+
10+
namespace Automattic\Jetpack\Jetpack_Mu_Wpcom\Blog_Privacy;
11+
12+
use Automattic\Jetpack\Jetpack_Mu_Wpcom;
13+
14+
require_once Jetpack_Mu_Wpcom::PKG_DIR . 'src/features/blog-privacy/blog-privacy.php';
15+
16+
/**
17+
* Tests for the functions defined in the Blog Privacy feature.
18+
*/
19+
class Blog_Privacy_Test extends \WorDBless\BaseTestCase {
20+
/**
21+
* Post-test suite actions.
22+
*/
23+
public static function tear_down_after_class() {
24+
\delete_option( 'wpcom_data_sharing_opt_out' );
25+
}
26+
27+
/**
28+
* Data provider for ->test_robots_txt().
29+
*
30+
* @return \Iterator
31+
*/
32+
public function provide_robots_txt(): \Iterator {
33+
$ai_blocks = <<<AI_BLOCKS
34+
User-agent: Amazonbot
35+
Disallow: /
36+
37+
User-agent: CCBot
38+
Disallow: /
39+
40+
User-agent: FacebookBot
41+
Disallow: /
42+
43+
User-agent: Google-Extended
44+
Disallow: /
45+
46+
User-agent: GPTBot
47+
Disallow: /
48+
49+
User-agent: omgili
50+
Disallow: /
51+
52+
User-agent: omgilibot
53+
Disallow: /
54+
55+
User-agent: SentiBot
56+
Disallow: /
57+
58+
User-agent: sentibot
59+
Disallow: /
60+
61+
AI_BLOCKS;
62+
63+
yield 'public, no discourage AI' => array(
64+
'blog_public' => '1',
65+
'wpcom_data_sharing_opt_out' => null,
66+
'expected' => 'TEST',
67+
);
68+
69+
yield 'public, discourage AI' => array(
70+
'blog_public' => '1',
71+
'wpcom_data_sharing_opt_out' => '1',
72+
'expected' => "TEST\n$ai_blocks",
73+
);
74+
75+
yield 'discourage search, no discourage AI' => array(
76+
'blog_public' => '0',
77+
'wpcom_data_sharing_opt_out' => null,
78+
'expected' => 'TEST',
79+
);
80+
81+
yield 'discourage search, discourage AI' => array(
82+
'blog_public' => '0',
83+
'wpcom_data_sharing_opt_out' => '1',
84+
'expected' => 'TEST', // We already discourage all user agents.
85+
);
86+
87+
yield 'private, no discourage AI' => array(
88+
'blog_public' => '-1',
89+
'wpcom_data_sharing_opt_out' => null,
90+
'expected' => 'TEST',
91+
);
92+
93+
yield 'private, discourage AI' => array(
94+
'blog_public' => '-1',
95+
'wpcom_data_sharing_opt_out' => '1',
96+
'expected' => 'TEST', // Private overrides wpcom_data_sharing_opt_out setting.
97+
);
98+
}
99+
100+
/**
101+
* Test for robots_txt hook.
102+
*/
103+
public function test_robots_txt_is_hooked() {
104+
$this->assertSame( 12, \has_filter( 'robots_txt', __NAMESPACE__ . '\\robots_txt' ) );
105+
}
106+
107+
/**
108+
* Tests for robots_txt().
109+
*
110+
* @dataProvider provide_robots_txt
111+
* @param mixed $blog_public The value of the blog_public option.
112+
* @param mixed|null $wpcom_data_sharing_opt_out The value of the wpcom_data_sharing_opt_out option (null if the option is not set).
113+
* @param string $expected The expected output for robots.txt.
114+
*/
115+
public function test_robots_txt( $blog_public, $wpcom_data_sharing_opt_out, string $expected ) {
116+
if ( null === $wpcom_data_sharing_opt_out ) {
117+
\delete_option( 'wpcom_data_sharing_opt_out' );
118+
} else {
119+
\update_option( 'wpcom_data_sharing_opt_out', $wpcom_data_sharing_opt_out );
120+
}
121+
122+
$this->assertSame( $expected, robots_txt( 'TEST', $blog_public ) );
123+
}
124+
}

‎projects/plugins/mu-wpcom-plugin/changelog/add-mu-wpcom-ai-robots-txt-blocks

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
Significance: patch
2+
Type: changed
3+
Comment: Updated composer.lock.
4+
5+

‎projects/plugins/mu-wpcom-plugin/composer.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,6 @@
4646
]
4747
},
4848
"config": {
49-
"autoloader-suffix": "d9d132a783958a00a2c7cccff60ca42d_jetpack_mu_wpcom_pluginⓥ2_0_21"
49+
"autoloader-suffix": "d9d132a783958a00a2c7cccff60ca42d_jetpack_mu_wpcom_pluginⓥ2_0_22_alpha"
5050
}
5151
}

‎projects/plugins/mu-wpcom-plugin/composer.lock

+2-2
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎projects/plugins/mu-wpcom-plugin/mu-wpcom-plugin.php

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
*
44
* Plugin Name: WordPress.com Features
55
* Description: Test plugin for the jetpack-mu-wpcom package
6-
* Version: 2.0.21
6+
* Version: 2.0.22-alpha
77
* Author: Automattic
88
* License: GPLv2 or later
99
* Text Domain: jetpack-mu-wpcom-plugin

‎projects/plugins/mu-wpcom-plugin/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"private": true,
33
"name": "@automattic/jetpack-mu-wpcom-plugin",
4-
"version": "2.0.21",
4+
"version": "2.0.22-alpha",
55
"description": "Test plugin for the jetpack-mu-wpcom package",
66
"homepage": "https://jetpack.com",
77
"bugs": {

0 commit comments

Comments
 (0)