Init project

.gitignore

@@ -0,0 +1,4 @@
+.vs/
+vendor/
+composer.lock
+test.php

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 量子复合态
+Copyright (c) 2020 Hyperzlib
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README-zh.md

@@ -0,0 +1,30 @@
+# Isekai AI Review
+[English](README.md)
+
+这个扩展必须和Moderation扩展一起用。
+
+通过AI审核的编辑会自动在Moderation中通过审核。
+
+如果你想要增加新的API接口，可以提交issue给我。
+
+## 使用方法
+先在阿里云注册：[https://www.aliyun.com/product/lvwang](https://www.aliyun.com/product/lvwang)
+
+然后安装Moderation扩展：[https://github.com/edwardspec/mediawiki-moderation](https://github.com/edwardspec/mediawiki-moderation)
+
+安装composer包：（如果你是在release页面下载的，可以忽略这一项）
+```php
+composer update
+```
+
+在```LocalSettings.php```中添加相关配置：
+```php
+wfLoadExtension('IsekaiAIReview');
+
+//配置部分
+$wgAIReviewEndpoint = 'cn-shanghai';
+$wgAIReviewAccessKeyId = '阿里云的Access key id';
+$wgAIReviewAccessKeySecret = '阿里云的Access key secret';
+$wgAIReviewBizType = 'isekaiwiki';
+$wgAIReviewRobotUID = 0; //在Moderation里显示的，执行审核操作的机器人账号的UID
+```

README.md

@@ -0,0 +1,29 @@
+# Isekai AI Review
+[中文文档](README-zh.md)
+
+This extension require mediawiki Moderation extension.
+Use AI to auto review revs in Moderation.
+
+If you want to add more AI Review API, you can submit a issue.
+
+## Useage
+First, register at Aliyun: [https://www.aliyun.com/product/lvwang](https://www.aliyun.com/product/lvwang)
+
+And then, install the Moderation extension: [https://github.com/edwardspec/mediawiki-moderation](https://github.com/edwardspec/mediawiki-moderation)
+
+Install composer packages (If you download the release, ignore it)
+```php
+composer update
+```
+
+Finally, add config in ```LocalSettings.php```:
+```php
+wfLoadExtension('IsekaiAIReview');
+
+//config
+$wgAIReviewEndpoint = 'cn-shanghai';
+$wgAIReviewAccessKeyId = '阿里云的Access key id';
+$wgAIReviewAccessKeySecret = '阿里云的Access key secret';
+$wgAIReviewBizType = 'isekaiwiki';
+$wgAIReviewRobotUID = 0; //The user account show in Moderation which approve revs
+```

composer.json

@@ -0,0 +1,14 @@
+{
+    "name": "hyperzlib/isekai-ai-review",
+	"type": "mediawiki-extension",
+    "require": {
+        "alibabacloud/sdk": "^1.8",
+        "paquettg/php-html-parser": "^2.2"
+    },
+    "authors": [
+        {
+            "name": "量子复合态",
+            "email": "hyperzlib@outlook.com"
+        }
+    ]
+}

extension.json

@@ -0,0 +1,57 @@
+{
+  "name": "IsekaiAIReview",
+  "author": "hyperzlib",
+  "url": "https://www.isekai.cn",
+  "descriptionmsg": "isekai-aireview-desc",
+  "version": "1.0.0",
+  "license-name": "MIT",
+  "type": "other",
+  "requires": {
+    "MediaWiki": ">= 1.31.0",
+    "extensions": {
+			"Moderation": ">= 1.5.0"
+		}
+  },
+  "ExtensionMessagesFiles": {
+    "IsekaiAIReviewAlias": "IsekaiAIReview.alias.php"
+  },
+  "MessagesDirs": {
+    "IsekaiAIReview": [
+      "i18n"
+    ]
+  },
+  "AutoloadClasses": {
+    "Isekai\\AIReview\\Hooks": "includes/Hooks.php",
+    "Isekai\\AIReview\\SectionSplitter": "includes/SectionSplitter.php",
+    "Isekai\\AIReview\\Utils": "includes/Utils.php",
+    "Isekai\\AIReview\\AliyunAIReview": "includes/AliyunAIReview.php",
+    "Isekai\\AIReview\\AIReviewJob": "includes/AIReviewJob.php",
+    "Isekai\\AIReview\\LogFormatter": "includes/LogFormatter.php"
+  },
+  "Hooks": {
+    "ModerationPending": [
+      "Isekai\\AIReview\\Hooks::onModerationPending"
+    ]
+  },
+  "JobClasses": {
+    "IsekaiAIReview": "Isekai\\AIReview\\AIReviewJob"
+  },
+  "LogTypes": [
+    "aireview"
+  ],
+  "LogActionsHandlers": {
+    "aireview/*": "Isekai\\AIReview\\LogFormatter"
+  },
+  "LogRestrictions": {
+		"aireview": "moderation"
+	},
+  "config": {
+    "AIReviewEndpoint": "cn-shanghai",
+    "AIReviewAccessKeyId": "",
+    "AIReviewAccessKeySecret": "",
+    "AIReviewBizType": null,
+    "AIReviewRobotUID": 1
+  },
+  "load_composer_autoloader": true,
+  "manifest_version": 1
+}

i18n/zh-hans.json

@@ -0,0 +1,21 @@
+{
+    "isekai-aireview-desc": "通过AI实现自动审核",
+    "log-name-aireview": "机器审核记录",
+    "log-description-aireview": "这里会显示经过机器审核的记录",
+    "logentry-aireview-approve": "AI已{{GENDER:$2|通过}}$4对页面$3的$5",
+    "logentry-aireview-reject": "AI已{{GENDER:$2|阻止}}$4对页面$3的$5，原因：$6",
+
+    "isekai-aireview-aliyun-reason-normal": "正常（不知道为什么会被拦截）",
+    "isekai-aireview-aliyun-reason-spam": "垃圾信息",
+    "isekai-aireview-aliyun-reason-ad": "广告",
+    "isekai-aireview-aliyun-reason-politics": "政治敏感",
+    "isekai-aireview-aliyun-reason-terrorism": "暴恐",
+    "isekai-aireview-aliyun-reason-abuse": "辱骂",
+    "isekai-aireview-aliyun-reason-porn": "色情",
+    "isekai-aireview-aliyun-reason-flood": "灌水",
+    "isekai-aireview-aliyun-reason-contraband": "违禁",
+    "isekai-aireview-aliyun-reason-meaningless": "无意义",
+    "isekai-aireview-aliyun-reason-customized": "违禁词",
+    "isekai-aireview-aliyun-reason-unknow": "未知: $1",
+    "isekai-aireview-aliyun-server-error": "服务器返回: $1"
+}

includes/AIReviewJob.php

@@ -0,0 +1,68 @@
+<?php
+namespace Isekai\AIReview;
+
+use Job;
+use MediaWiki\MediaWikiServices;
+use MediaWiki\Moderation\AddLogEntryConsequence;
+use Title;
+use User;
+use RequestContext;
+use ModerationViewableEntry;
+
+class AIReviewJob extends Job {
+    public function __construct(Title $title, array $params){
+        parent::__construct('IsekaiAIReview', $title, $params);
+    }
+
+    /**
+     * 运行job，开始进行AI审核
+     */
+    public function run(){
+        global $wgAIReviewRobotUID;
+
+        $dbr = wfGetDB(DB_REPLICA);
+
+        $robotUser = User::newFromId($wgAIReviewRobotUID);
+
+        $modid = $this->params['mod_id'];
+        $modUser = $dbr->selectField('moderation', 'mod_user', ['mod_id' => $modid], __METHOD__);
+
+        $services = MediaWikiServices::getInstance();
+        $entryFactory = $services->getService('Moderation.EntryFactory');
+        $consequenceManager = $services->getService('Moderation.ConsequenceManager');
+
+        /** @var ModerationViewableEntry $contentEntry  */
+        $contentEntry = $entryFactory->findViewableEntry($modid);
+        $title = $contentEntry->getTitle();
+
+        $context = RequestContext::getMain();
+        $context->setTitle($title);
+        //获取diff内容
+        $diffHtml = $contentEntry->getDiffHTML($context);
+        //取出增加的文本内容
+        $addedText = Utils::getDiffAddedLines($diffHtml);
+        if(strlen($addedText) > 0){
+            //开始进行AI审核
+            $reviewer = new AliyunAIReview();
+            $result = $reviewer->reviewText($addedText);
+            if(!$result['pass']){ //审核不通过
+                wfDebugLog(
+                    'isekai-aireview',
+                    'Reject revision on: ' . $title->getText() . ', reason: ' . Utils::getReadableReason($result['reason'])
+                );
+                Utils::addAIReviewLog('reject', $robotUser, $modUser, $title, $modid, $result['reason']);
+                return true;
+            }
+        }
+
+        //审核通过
+        wfDebugLog(
+            'isekai-aireview',
+            'Approve revision on: ' . $title->getText()
+        );
+        Utils::addAIReviewLog('approve', $robotUser, $modUser, $title, $modid);
+        $approveEntry = $entryFactory->findApprovableEntry($modid);
+        $approveEntry->approve($robotUser);
+        return true;
+    }
+}

includes/AliyunAIReview.php

@@ -0,0 +1,85 @@
+<?php
+namespace Isekai\AIReview;
+
+use AlibabaCloud\Client\AlibabaCloud;
+use AlibabaCloud\Green\Green;
+use Exception;
+
+class AliyunAIReview {
+    private const MAX_LENGTH = 10000;
+
+    public function __construct(){
+        global $wgAIReviewEndpoint, $wgAIReviewAccessKeyId, $wgAIReviewAccessKeySecret;
+        AlibabaCloud::accessKeyClient($wgAIReviewAccessKeyId, $wgAIReviewAccessKeySecret)
+            ->regionId($wgAIReviewEndpoint)
+            ->asDefaultClient();
+    }
+
+    public function reviewText($text){
+        $reqData = $this->buildRequestData($text);
+        $response = $this->doRequest($reqData);
+        return $response;
+    }
+
+    public function buildRequestData($text){
+        global $wgAIReviewBizType;
+
+        $reqData = [
+            'scenes' => ['antispam'],
+            'tasks' => $this->buildTasks($text),
+        ];
+
+        if($wgAIReviewBizType) $reqData['bizType'] = $wgAIReviewBizType;
+        return $reqData;
+    }
+
+    public function buildTasks($text){
+        $splitter = new SectionSplitter($text, self::MAX_LENGTH);
+        $chunkList = $splitter->getChunkList();
+        $taskList = [];
+        foreach($chunkList as $chunk){
+            $task = [
+                'dataId' => uniqid(),
+                'content' => $chunk,
+            ];
+            $taskList[] = $task;
+        };
+        unset($chunkList);
+        return $taskList;
+    }
+
+    public function doRequest($requestData){
+        $textScan = Green::v20180509()->textScan();
+        $response = $textScan->setMethod('POST')->setAcceptFormat('JSON')->setContent(json_encode($requestData))->request();
+
+        if($response->getReasonPhrase() === 'OK'){
+            return $this->parseResponse($response->toArray());
+        } else {
+            return ['pass' => false, 'reason' => wfMessage('isekai-aireview-aliyun-server-error', $response->getStatusCode())->escaped()];
+        }
+    }
+
+    public function parseResponse($response){
+        if($response['code'] !== 200)
+            return ['pass' => false, 'reason' => wfMessage('isekai-aireview-aliyun-server-error', $response['code'])->escaped()];
+        
+        $pass = true;
+        $reasons = [];
+        foreach($response['data'] as $task){
+            if(is_array($task['results'])){
+                foreach($task['results'] as $result){
+                    if($result['suggestion'] !== 'pass'){
+                        $pass = false;
+                        foreach($result['details'] as $detail){
+                            $reason = $detail['label'];
+                            if(!in_array($reason, $reasons)){
+                                $reasons[] = $reason;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        return ['pass' => $pass, 'reason' => $reasons];
+    }
+}

includes/Hooks.php

@@ -0,0 +1,13 @@
+<?php
+namespace Isekai\AIReview;
+
+use JobQueueGroup;
+use Title;
+
+class Hooks {
+    public static function onModerationPending($fields, $modid){
+        //加入审核队列
+        $job = new AIReviewJob(Title::newFromText($fields['mod_title']), ['mod_id' => $modid]);
+        JobQueueGroup::singleton()->push($job);
+    }
+}

includes/LogFormatter.php

@@ -0,0 +1,70 @@
+<?php
+namespace Isekai\AIReview;
+
+use LogFormatter as GlobalLogFormatter;
+use SpecialPage;
+use Message;
+use Linker;
+use Title;
+use User;
+
+class LogFormatter extends GlobalLogFormatter {
+    public function getMessageParameters(){
+        $params = parent::getMessageParameters();
+
+        $type = $this->entry->getSubtype();
+        $entryParams = $this->entry->getParameters();
+        $linkRenderer = $this->getLinkRenderer();
+        
+        switch($type){
+            case 'approve':
+                $modId = $entryParams['modid'];
+
+                $user = User::newFromId($entryParams['moduser']);
+                $userLink = Linker::userLink( $user->getId(), $user->getName() );
+                $params[3] = Message::rawParam( $userLink );
+
+                $link = $linkRenderer->makeKnownLink(
+                    SpecialPage::getTitleFor( 'Moderation' ),
+                    $this->msg( 'moderation-log-change' )->params( $modId )->text(),
+                    [ 'title' => $this->msg( 'tooltip-moderation-rejected-change' )->plain() ],
+                    [ 'modaction' => 'show', 'modid' => $modId ]
+                );
+                $params[4] = Message::rawParam( $link );
+
+                break;
+            case 'reject':
+                $modId = $entryParams['modid'];
+
+                $user = User::newFromId($entryParams['moduser']);
+                $userLink = Linker::userLink( $user->getId(), $user->getName() );
+                $params[3] = Message::rawParam( $userLink );
+
+                $link = $linkRenderer->makeKnownLink(
+                    SpecialPage::getTitleFor( 'Moderation' ),
+                    $this->msg( 'moderation-log-change' )->params( $modId )->text(),
+                    [ 'title' => $this->msg( 'tooltip-moderation-rejected-change' )->plain() ],
+                    [ 'modaction' => 'show', 'modid' => $modId ]
+                );
+                $params[4] = Message::rawParam( $link );
+
+                $params[5] = Utils::getReadableReason($entryParams['reason']);
+                break;
+        }
+        return $params;
+    }
+
+    public function getPreloadTitles() {
+		$type = $this->entry->getSubtype();
+		$params = $this->entry->getParameters();
+
+		$titles = [];
+
+        if ( $params['moduser'] ) { # Not anonymous
+            $user = User::newFromId($params['moduser']);
+            $titles[] = Title::makeTitle( NS_USER, $user->getName() );
+        }
+
+		return $titles;
+	}
+}

includes/SectionSplitter.php

@@ -0,0 +1,78 @@
+<?php
+namespace Isekai\AIReview;
+
+class SectionSplitter {
+    private $chunkList = [''];
+    private $chunkListSeek = 0;
+    private $bufferLength = 0;
+    private $maxLength;
+
+    public function __construct($text, $maxLength = 10000){
+        $this->maxLength = $maxLength;
+        $this->splitLine($text);
+    }
+
+    /* 将文本推入chunk列表 */
+    public function push($chunk){
+        $chunkLength = mb_strlen($chunk, 'UTF-8');
+        if($this->bufferLength + $chunkLength > $this->maxLength){ //满一万字
+            $this->chunkListSeek ++;
+            $this->chunkList[$this->chunkListSeek] = $chunk;
+            $this->bufferLength = $chunkLength;
+        } else { //没满一万字，接着塞
+            $this->chunkList[$this->chunkListSeek] .= $chunk;
+            $this->bufferLength += $chunkLength;
+        }
+    }
+
+    /**
+     * 按照行来拆分
+     */
+    public function splitLine($text){
+        $text = str_replace("\r\n", "\n", $text);
+        $lines = explode("\n", $text);
+        foreach($lines as $line){
+            if(empty($line)) continue;
+
+            $line .= "\n";
+            if(mb_strlen($line, 'UTF-8') > $this->maxLength){ //见鬼，这个人怎么能写一万字不换行
+                $this->splitSentence($line);
+            } else {
+                $this->push($line);
+            }
+        }
+    }
+
+    /**
+     * 按照句子来拆分
+     */
+    public function splitSentence($text){ //我就不信一句话能一万字
+        $sentences = explode("\0", preg_replace('/(。|\\.)/', "$1\0", $text));
+        foreach($sentences as $sentence){
+            if(mb_strlen($sentence, 'UTF-8') > $this->maxLength){ //一句话能说一万字吗？
+                $this->forceSplit($sentence);
+            } else {
+                $this->push($sentence);
+            }
+        }
+    }
+
+    /**
+     * 强制拆分
+     */
+    public function forceSplit($text){
+        $len = mb_strlen($text, 'UTF-8');
+        $times = ceil($len / $this->maxLength);
+        for($i = 0; $i < $times; $i ++){
+            $startPos = $i * $this->maxLength;
+            $sentenceLen = min($len - 1 - $i * $startPos, $this->maxLength);
+            $sentence = substr($text, $startPos, $sentenceLen);
+            
+            $this->push($sentence);
+        }
+    }
+
+    public function getChunkList(){
+        return $this->chunkList;
+    }
+}

includes/Utils.php

@@ -0,0 +1,56 @@
+<?php
+namespace Isekai\AIReview;
+
+use ManualLogEntry;
+use PHPHtmlParser\Dom;
+
+class Utils {
+    public static function getDiffAddedLines($diffHtml){
+        $dom = new Dom();
+        $dom->load($diffHtml);
+        $lines = [];
+
+        if($addedLineDomList = $dom->find('.diff-addedline')){
+            /** @var \PHPHtmlParser\Dom\HtmlNode $addedLineDom */
+            foreach($addedLineDomList as $addedLineDom){
+                $lines[] = strip_tags($addedLineDom->innerHtml);
+            }
+        }
+
+        return trim(implode("\n", $lines));
+    }
+
+    public static function getReadableReason($reasons){
+        $allowedReasons = ['spam', 'ad', 'politics', 'terrorism', 'abuse', 'porn', 'flood', 'contraband', 'meaningless', 'customized', 'normal'];
+
+        if(is_string($reasons)) return $reasons;
+
+        $readableReasons = [];
+        foreach($reasons as $reason){
+            if(in_array($reason, $allowedReasons)){
+                $readableReasons[] = wfMessage('isekai-aireview-aliyun-reason-' . $reason)->escaped();
+            } else {
+                $readableReasons[] = wfMessage('isekai-aireview-aliyun-reason-unknow', $reason)->escaped();
+            }
+        }
+
+        return implode(', ', $readableReasons);
+    }
+
+    public static function addAIReviewLog($event, $robotUser, $modUser, $title, $modid, $reason = null){
+        $entry = new ManualLogEntry('aireview', $event);
+        $entry->setPerformer($robotUser);
+        $entry->setTarget($title);
+
+        $param = [
+            'modid' => $modid,
+            'moduser' => $modUser,
+        ];
+        if($reason){
+            $param['reason'] = $reason;
+        }
+
+        $entry->setParameters($param);
+        $entry->insert();
+    }
+}