jakob/friendica
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

The "term" table is removed

Browse source
This commit is contained in:
Michael 2020-05-15 06:50:20 +00:00
parent f58d42e317
commit 7a82171bfe
8 changed files with 68 additions and 265 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
database.sql
View file

@ -1,6 +1,6 @@
-- ------------------------------------------
-- Friendica 2020.06-dev (Red Hot Poker)
-- DB_UPDATE_VERSION 1347
-- DB_UPDATE_VERSION 1348
-- ------------------------------------------
@ -107,7 +107,9 @@ CREATE TABLE IF NOT EXISTS `auth_codes` (
`redirect_uri` varchar(200) NOT NULL DEFAULT '' COMMENT '',
`expires` int NOT NULL DEFAULT 0 COMMENT '',
`scope` varchar(250) NOT NULL DEFAULT '' COMMENT '',
PRIMARY KEY(`id`)
PRIMARY KEY(`id`),
INDEX `client_id` (`client_id`),
CONSTRAINT `auth_codes-client_id-clients-client_id` FOREIGN KEY (`client_id`) REFERENCES `clients` (`client_id`) ON UPDATE RESTRICT ON DELETE CASCADE
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='OAuth usage';
--
@ -304,7 +306,8 @@ CREATE TABLE IF NOT EXISTS `conversation` (
CREATE TABLE IF NOT EXISTS `diaspora-interaction` (
`uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri',
`interaction` mediumtext COMMENT 'The Diaspora interaction',
PRIMARY KEY(`uri-id`)
PRIMARY KEY(`uri-id`),
CONSTRAINT `diaspora-interaction-uri-id-item-uri-id` FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Signed Diaspora Interaction';
--
@ -668,7 +671,13 @@ CREATE TABLE IF NOT EXISTS `item` (
INDEX `icid` (`icid`),
INDEX `iaid` (`iaid`),
INDEX `psid_wall` (`psid`,`wall`),
INDEX `uri-id` (`uri-id`)
INDEX `uri-id` (`uri-id`),
INDEX `parent-uri-id` (`parent-uri-id`),
INDEX `thr-parent-id` (`thr-parent-id`),
CONSTRAINT `item-uri-id-item-uri-id` FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
CONSTRAINT `item-parent-uri-id-item-uri-id` FOREIGN KEY (`parent-uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
CONSTRAINT `item-thr-parent-id-item-uri-id` FOREIGN KEY (`thr-parent-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
CONSTRAINT `item-psid-permissionset-id` FOREIGN KEY (`psid`) REFERENCES `permissionset` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Structure for all posts';
--
@ -683,7 +692,8 @@ CREATE TABLE IF NOT EXISTS `item-activity` (
PRIMARY KEY(`id`),
UNIQUE INDEX `uri-hash` (`uri-hash`),
INDEX `uri` (`uri`(191)),
INDEX `uri-id` (`uri-id`)
INDEX `uri-id` (`uri-id`),
CONSTRAINT `item-activity-uri-id-item-uri-id` FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Activities for items';
--
@ -713,7 +723,8 @@ CREATE TABLE IF NOT EXISTS `item-content` (
UNIQUE INDEX `uri-plink-hash` (`uri-plink-hash`),
INDEX `uri` (`uri`(191)),
INDEX `plink` (`plink`(191)),
INDEX `uri-id` (`uri-id`)
INDEX `uri-id` (`uri-id`),
CONSTRAINT `item-content-uri-id-item-uri-id` FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Content for all posts';
--
@ -839,7 +850,8 @@ CREATE TABLE IF NOT EXISTS `notify-threads` (
`master-parent-uri-id` int unsigned COMMENT 'Item-uri id of the parent of the related post',
`parent-item` int unsigned NOT NULL DEFAULT 0 COMMENT '',
`receiver-uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'User id',
PRIMARY KEY(`id`)
PRIMARY KEY(`id`),
INDEX `master-parent-uri-id` (`master-parent-uri-id`)
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='';
--
@ -1080,7 +1092,8 @@ CREATE TABLE IF NOT EXISTS `profile_field` (
PRIMARY KEY(`id`),
INDEX `uid` (`uid`),
INDEX `order` (`order`),
INDEX `psid` (`psid`)
INDEX `psid` (`psid`),
CONSTRAINT `profile_field-psid-permissionset-id` FOREIGN KEY (`psid`) REFERENCES `permissionset` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Custom profile fields';
--
@ -1139,29 +1152,6 @@ CREATE TABLE IF NOT EXISTS `session` (
INDEX `expire` (`expire`)
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='web session storage';
--
-- TABLE term
--
CREATE TABLE IF NOT EXISTS `term` (
`tid` int unsigned NOT NULL auto_increment COMMENT '',
`oid` int unsigned NOT NULL DEFAULT 0 COMMENT '',
`otype` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '',
`type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '',
`term` varchar(255) NOT NULL DEFAULT '' COMMENT '',
`url` varchar(255) NOT NULL DEFAULT '' COMMENT '',
`guid` varchar(255) NOT NULL DEFAULT '' COMMENT '',
`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
`received` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
`global` boolean NOT NULL DEFAULT '0' COMMENT '',
`uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'User id',
PRIMARY KEY(`tid`),
INDEX `term_type` (`term`(64),`type`),
INDEX `oid_otype_type_term` (`oid`,`otype`,`type`,`term`(32)),
INDEX `uid_otype_type_term_global_created` (`uid`,`otype`,`type`,`term`(32),`global`,`created`),
INDEX `uid_otype_type_url` (`uid`,`otype`,`type`,`url`(64)),
INDEX `guid` (`guid`(64))
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='item taxonomy (categories, tags, etc.) table';
--
-- TABLE tag
--
@ -1183,7 +1173,9 @@ CREATE TABLE IF NOT EXISTS `post-category` (
`type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '',
`tid` int unsigned NOT NULL DEFAULT 0 COMMENT '',
PRIMARY KEY(`uri-id`,`uid`,`type`,`tid`),
INDEX `uri-id` (`tid`)
INDEX `uri-id` (`tid`),
CONSTRAINT `post-category-uri-id-item-uri-id` FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
CONSTRAINT `post-category-tid-tag-id` FOREIGN KEY (`tid`) REFERENCES `tag` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='post relation to categories';
--
@ -1201,7 +1193,8 @@ CREATE TABLE IF NOT EXISTS `post-delivery-data` (
`legacy_dfrn` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries via legacy DFRN',
`diaspora` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries via Diaspora',
`ostatus` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries via OStatus',
PRIMARY KEY(`uri-id`)
PRIMARY KEY(`uri-id`),
CONSTRAINT `post-delivery-data-uri-id-item-uri-id` FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Delivery data for items';
--
@ -1213,8 +1206,11 @@ CREATE TABLE IF NOT EXISTS `post-tag` (
`tid` int unsigned NOT NULL DEFAULT 0 COMMENT '',
`cid` int unsigned NOT NULL DEFAULT 0 COMMENT 'Contact id of the mentioned public contact',
PRIMARY KEY(`uri-id`,`type`,`tid`,`cid`),
INDEX `uri-id` (`tid`),
INDEX `cid` (`tid`)
INDEX `tid` (`tid`),
INDEX `cid` (`cid`),
CONSTRAINT `post-tag-uri-id-item-uri-id` FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
CONSTRAINT `post-tag-tid-tag-id` FOREIGN KEY (`tid`) REFERENCES `tag` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT,
CONSTRAINT `post-tag-cid-contact-id` FOREIGN KEY (`cid`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='post relation to tags';
--
@ -1270,7 +1266,9 @@ CREATE TABLE IF NOT EXISTS `tokens` (
`expires` int NOT NULL DEFAULT 0 COMMENT '',
`scope` varchar(200) NOT NULL DEFAULT '' COMMENT '',
`uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'User id',
PRIMARY KEY(`id`)
PRIMARY KEY(`id`),
INDEX `client_id` (`client_id`),
CONSTRAINT `tokens-client_id-clients-client_id` FOREIGN KEY (`client_id`) REFERENCES `clients` (`client_id`) ON UPDATE RESTRICT ON DELETE CASCADE
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='OAuth usage';
--
@ -1367,7 +1365,7 @@ CREATE TABLE IF NOT EXISTS `user-item` (
-- TABLE verb
--
CREATE TABLE IF NOT EXISTS `verb` (
`id` int unsigned NOT NULL auto_increment,
`id` smallint unsigned NOT NULL auto_increment,
`name` varchar(100) NOT NULL DEFAULT '' COMMENT '',
PRIMARY KEY(`id`)
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Activity Verbs';

31
src/Database/PostUpdate.php
View file

@ -30,7 +30,6 @@ use Friendica\Model\ItemURI;
use Friendica\Model\PermissionSet;
use Friendica\Model\Post\Category;
use Friendica\Model\Tag;
use Friendica\Model\Term;
use Friendica\Model\UserItem;
use Friendica\Model\Verb;
use Friendica\Util\Strings;
@ -43,6 +42,9 @@ use Friendica\Util\Strings;
*/
class PostUpdate
{
// Needed for the helper function to read from the legacy term table
const OBJECT_TYPE_POST = 1;
/**
* Calls the post update functions
*/
@ -731,6 +733,31 @@ class PostUpdate
return false;
}
/**
* Generates the legacy item.file field string from an item ID.
* Includes only file and category terms.
*
* @param int $item_id
* @return string
* @throws \Exception
*/
private static function fileTextFromItemId($item_id)
{
$file_text = '';
$condition = ['otype' => self::OBJECT_TYPE_POST, 'oid' => $item_id, 'type' => [Category::FILE, Category::CATEGORY]];
$tags = DBA::selectToArray('term', ['type', 'term', 'url'], $condition);
foreach ($tags as $tag) {
if ($tag['type'] == Category::CATEGORY) {
$file_text .= '<' . $tag['term'] . '>';
} else {
$file_text .= '[' . $tag['term'] . ']';
}
}
return $file_text;
}
/**
* Fill the "tag" table with tags and mentions from the "term" table
*
@ -765,7 +792,7 @@ class PostUpdate
continue;
}
$file = Term::fileTextFromItemId($term['oid']);
$file = self::fileTextFromItemId($term['oid']);
if (!empty($file)) {
Category::storeTextByURIId($item['uri-id'], $item['uid'], $file);
}

115
src/Model/Term.php
View file

@ -1,115 +0,0 @@
<?php
/**
* @copyright Copyright (C) 2020, Friendica
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
namespace Friendica\Model;
use Friendica\Database\DBA;
/**
* Class Term
*
* This Model class handles term table interactions.
* This tables stores relevant terms related to posts, photos and searches, like hashtags, mentions and
* user-applied categories.
*/
class Term
{
const UNKNOWN = 0;
const CATEGORY = 3;
const FILE = 5;
const OBJECT_TYPE_POST = 1;
/**
* Generates the legacy item.file field string from an item ID.
* Includes only file and category terms.
*
* @param int $item_id
* @return string
* @throws \Exception
*/
public static function fileTextFromItemId($item_id)
{
$file_text = '';
$condition = ['otype' => self::OBJECT_TYPE_POST, 'oid' => $item_id, 'type' => [self::FILE, self::CATEGORY]];
$tags = DBA::selectToArray('term', ['type', 'term', 'url'], $condition);
foreach ($tags as $tag) {
if ($tag['type'] == self::CATEGORY) {
$file_text .= '<' . $tag['term'] . '>';
} else {
$file_text .= '[' . $tag['term'] . ']';
}
}
return $file_text;
}
/**
* Inserts new terms for the provided item ID based on the legacy item.file field BBCode content.
* Deletes all previous file terms for the same item ID.
*
* @param integer $item_id item id
* @param $files
* @return void
* @throws \Exception
*/
public static function insertFromFileFieldByItemId($item_id, $files)
{
$message = Item::selectFirst(['uid', 'deleted'], ['id' => $item_id]);
if (!DBA::isResult($message)) {
return;
}
// Clean up all tags
DBA::delete('term', ['otype' => self::OBJECT_TYPE_POST, 'oid' => $item_id, 'type' => [self::FILE, self::CATEGORY]]);
if ($message["deleted"]) {
return;
}
$message['file'] = $files;
if (preg_match_all("/\[(.*?)\]/ism", $message["file"], $files)) {
foreach ($files[1] as $file) {
DBA::insert('term', [
'uid' => $message["uid"],
'oid' => $item_id,
'otype' => self::OBJECT_TYPE_POST,
'type' => self::FILE,
'term' => $file
]);
}
}
if (preg_match_all("/\<(.*?)\>/ism", $message["file"], $files)) {
foreach ($files[1] as $file) {
DBA::insert('term', [
'uid' => $message["uid"],
'oid' => $item_id,
'otype' => self::OBJECT_TYPE_POST,
'type' => self::CATEGORY,
'term' => $file
]);
}
}
}
}

1
src/Module/Admin/Site.php
View file

@ -103,7 +103,6 @@ class Site extends BaseAdmin
// update tables
// update profile links in the format "http://server.tld"
update_table($a, "profile", ['photo', 'thumb'], $old_url, $new_url);
update_table($a, "term", ['url'], $old_url, $new_url);
update_table($a, "contact", ['photo', 'thumb', 'micro', 'url', 'nurl', 'alias', 'request', 'notify', 'poll', 'confirm', 'poco', 'avatar'], $old_url, $new_url);
update_table($a, "gcontact", ['url', 'nurl', 'photo', 'server_url', 'notify', 'alias'], $old_url, $new_url);
update_table($a, "item", ['owner-link', 'author-link', 'body', 'plink', 'tag'], $old_url, $new_url);

27
src/Worker/DBClean.php
View file

@ -227,31 +227,8 @@ class DBClean {
// The legacy functionality had been removed
DI::config()->set('system', 'finished-dbclean-6', true);
} elseif ($stage == 7) {
$last_id = DI::config()->get('system', 'dbclean-last-id-7', 0);
Logger::log("Deleting orphaned data from term table. Last ID: ".$last_id);
$r = DBA::p("SELECT `oid`, `tid` FROM `term`
WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `term`.`oid`) AND `tid` >= ?
ORDER BY `tid` LIMIT ?", $last_id, $limit);
$count = DBA::numRows($r);
if ($count > 0) {
Logger::log("found term orphans: ".$count);
while ($orphan = DBA::fetch($r)) {
$last_id = $orphan["tid"];
DBA::delete('term', ['oid' => $orphan["oid"]]);
}
Worker::add(PRIORITY_MEDIUM, 'DBClean', 7, $last_id);
} else {
Logger::log("No term orphans found");
}
DBA::close($r);
Logger::log("Done deleting ".$count." orphaned data from term table. Last ID: ".$last_id);
DI::config()->set('system', 'dbclean-last-id-7', $last_id);
if ($count < $limit) {
DI::config()->set('system', 'finished-dbclean-7', true);
}
// The legacy functionality had been removed
DI::config()->set('system', 'finished-dbclean-7', true);
} elseif ($stage == 8) {
if ($days <= 0) {
return;

60
src/Worker/TagUpdate.php
View file

@ -1,60 +0,0 @@
<?php
/**
* @copyright Copyright (C) 2020, Friendica
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
namespace Friendica\Worker;
use Friendica\Core\Logger;
use Friendica\Database\DBA;
use Friendica\Model\Term;
class TagUpdate
{
public static function execute()
{
$messages = DBA::p("SELECT `oid`,`item`.`guid`, `item`.`created`, `item`.`received` FROM `term` INNER JOIN `item` ON `item`.`id`=`term`.`oid` WHERE `term`.`otype` = 1 AND `term`.`guid` = ''");
Logger::log('fetched messages: ' . DBA::numRows($messages));
while ($message = DBA::fetch($messages)) {
if ($message['uid'] == 0) {
$global = true;
DBA::update('term', ['global' => true], ['otype' => Term::OBJECT_TYPE_POST, 'guid' => $message['guid']]);
} else {
$global = (DBA::count('term', ['uid' => 0, 'otype' => Term::OBJECT_TYPE_POST, 'guid' => $message['guid']]) > 0);
}
$fields = ['guid' => $message['guid'], 'created' => $message['created'],
'received' => $message['received'], 'global' => $global];
DBA::update('term', $fields, ['otype' => Term::OBJECT_TYPE_POST, 'oid' => $message['oid']]);
}
DBA::close($messages);
$messages = DBA::select('item', ['guid'], ['uid' => 0]);
Logger::log('fetched messages: ' . DBA::numRows($messages));
while ($message = DBA::fetch($messages)) {
DBA::update('item', ['global' => true], ['guid' => $message['guid']]);
}
DBA::close($messages);
}
}

24
static/dbstructure.config.php
View file

@ -1260,30 +1260,6 @@ return [
"expire" => ["expire"],
]
],
"term" => [
"comment" => "item taxonomy (categories, tags, etc.) table",
"fields" => [
"tid" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "comment" => ""],
"oid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["item" => "id"], "comment" => ""],
"otype" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => ""],
"type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => ""],
"term" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
"url" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
"guid" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
"created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""],
"received" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""],
"global" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""],
"uid" => ["type" => "mediumint unsigned", "not null" => "1", "default" => "0", "relation" => ["user" => "uid"], "comment" => "User id"],
],
"indexes" => [
"PRIMARY" => ["tid"],
"term_type" => ["term(64)", "type"],
"oid_otype_type_term" => ["oid", "otype", "type", "term(32)"],
"uid_otype_type_term_global_created" => ["uid", "otype", "type", "term(32)", "global", "created"],
"uid_otype_type_url" => ["uid", "otype", "type", "url(64)"],
"guid" => ["guid(64)"],
]
],
"tag" => [
"comment" => "tags and mentions",
"fields" => [

3
update.php
View file

@ -70,7 +70,8 @@ function update_1181()
{
// Fill the new fields in the term table.
Worker::add(PRIORITY_LOW, "TagUpdate");
// deactivated, the "term" table is deprecated
// Worker::add(PRIORITY_LOW, "TagUpdate");
return Update::SUCCESS;
}